Compare commits
2 Commits
main
...
feature/ci
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bd89fcf4f3 | ||
|
|
0d7f5d2a24 |
25
.gitignore
vendored
25
.gitignore
vendored
@ -1,27 +1,2 @@
|
||||
*.md
|
||||
!README.md
|
||||
!knowledge/**/*.md
|
||||
!services/comms/knowledge/**/*.md
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
build/
|
||||
test-results/
|
||||
artifacts/
|
||||
.venv
|
||||
.venv-ci
|
||||
tmp/
|
||||
.mainfix/
|
||||
.terraform/
|
||||
**/.terraform/
|
||||
*.tfvars
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
crash.log
|
||||
terraform/atlas/generated/
|
||||
|
||||
# Local demo credentials (never commit)
|
||||
scripts/ops/hermes_demo.env
|
||||
scripts/ops/hermes_triage_demo.env
|
||||
|
||||
512
Jenkinsfile
vendored
512
Jenkinsfile
vendored
@ -1,512 +0,0 @@
|
||||
// Mirror of ci/Jenkinsfile.titan-iac for multibranch discovery.
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-06
|
||||
- titan-11
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-13
|
||||
- titan-15
|
||||
- titan-17
|
||||
- titan-19
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
jenkins/jenkins-jenkins-agent: "true"
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
|
||||
resources:
|
||||
requests:
|
||||
cpu: "25m"
|
||||
memory: "256Mi"
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python:3.12-slim
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
- name: quality-tools
|
||||
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
- name: semgrep
|
||||
image: semgrep/semgrep:1.171.0
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
"""
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PIP_DISABLE_PIP_VERSION_CHECK = '1'
|
||||
PYTHONUNBUFFERED = '1'
|
||||
SUITE_NAME = 'titan_iac'
|
||||
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
|
||||
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
|
||||
SONARQUBE_PROJECT_KEY = 'titan_iac'
|
||||
SONARQUBE_TOKEN = credentials('sonarqube-token')
|
||||
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
|
||||
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
|
||||
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
|
||||
QUALITY_GATE_SEMGREP_ENFORCE = '0'
|
||||
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
|
||||
QUALITY_GATE_IRONBANK_ENFORCE = '1'
|
||||
QUALITY_GATE_IRONBANK_REQUIRED = '0'
|
||||
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
|
||||
}
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Install deps') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends git ca-certificates
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
fi
|
||||
pip install --no-cache-dir -r ci/requirements.txt
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Prepare local quality evidence') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
python3 -m testing.quality_gate --profile local --build-dir build
|
||||
local_quality_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect Semgrep evidence') {
|
||||
steps {
|
||||
container('semgrep') {
|
||||
sh '''#!/bin/sh
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
|
||||
semgrep_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
|
||||
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect SonarQube evidence') {
|
||||
steps {
|
||||
container('quality-tools') {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
args=(
|
||||
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
|
||||
"-Dsonar.login=${SONARQUBE_TOKEN}"
|
||||
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.sources=."
|
||||
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
|
||||
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
|
||||
)
|
||||
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
|
||||
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
|
||||
set +e
|
||||
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
|
||||
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
|
||||
token = os.getenv('SONARQUBE_TOKEN', '').strip()
|
||||
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
|
||||
|
||||
payload = {
|
||||
"status": "ERROR",
|
||||
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
|
||||
}
|
||||
if host and project_key:
|
||||
task_file = Path('.scannerwork/report-task.txt')
|
||||
task_id = ''
|
||||
if task_file.exists():
|
||||
for line in task_file.read_text(encoding='utf-8').splitlines():
|
||||
key, _, value = line.partition('=')
|
||||
if key == 'ceTaskId':
|
||||
task_id = value.strip()
|
||||
break
|
||||
if task_id:
|
||||
ce_query = urllib.parse.urlencode({"id": task_id})
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
|
||||
if token:
|
||||
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
||||
ce_request.add_header("Authorization", f"Basic {encoded}")
|
||||
try:
|
||||
with urllib.request.urlopen(ce_request, timeout=12) as response:
|
||||
ce_payload = json.loads(response.read().decode("utf-8"))
|
||||
except Exception:
|
||||
time.sleep(3)
|
||||
continue
|
||||
status = str(ce_payload.get("task", {}).get("status", "")).upper()
|
||||
if status in {"SUCCESS", "FAILED", "CANCELED"}:
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
query = urllib.parse.urlencode({"projectKey": project_key})
|
||||
request = urllib.request.Request(
|
||||
f"{host}/api/qualitygates/project_status?{query}",
|
||||
method="GET",
|
||||
)
|
||||
if token:
|
||||
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
||||
request.add_header("Authorization", f"Basic {encoded}")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=12) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
payload = {"status": "ERROR", "error": str(exc)}
|
||||
|
||||
with open(report_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, sort_keys=True)
|
||||
handle.write("\\n")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect IronBank evidence') {
|
||||
steps {
|
||||
container('quality-tools') {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
set +e
|
||||
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-check-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
|
||||
trivy_rc=$?
|
||||
set -e
|
||||
if [ "${trivy_rc}" -ne 0 ]; then
|
||||
rm -f build/trivy-fs.json
|
||||
cat > build/ironbank-compliance.json <<EOF
|
||||
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy scan failed","trivy_rc":${trivy_rc}}
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -s build/trivy-fs.json ]; then
|
||||
cat > build/ironbank-compliance.json <<EOF
|
||||
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
if [ -s build/trivy-fs.json ]; then
|
||||
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
|
||||
exit 0
|
||||
fi
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
|
||||
if report_path.exists():
|
||||
raise SystemExit(0)
|
||||
|
||||
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
|
||||
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
|
||||
payload = {
|
||||
"status": status or "unknown",
|
||||
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
|
||||
}
|
||||
payload = {k: v for k, v in payload.items() if v is not None}
|
||||
if "status" not in payload:
|
||||
payload["status"] = "unknown"
|
||||
payload["note"] = (
|
||||
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
|
||||
"or write build/ironbank-compliance.json in image-building repos."
|
||||
)
|
||||
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Run quality gate') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
python3 -m testing.quality_gate --profile jenkins --build-dir build
|
||||
quality_gate_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Publish test metrics') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
export JUNIT_GLOB='build/junit-*.xml'
|
||||
export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
|
||||
export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
|
||||
python3 ci/scripts/publish_test_metrics.py
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Enforce quality gate') {
|
||||
steps {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
|
||||
fail=0
|
||||
if [ "${gate_rc}" -ne 0 ]; then
|
||||
echo "quality gate failed with rc=${gate_rc}" >&2
|
||||
fail=1
|
||||
fi
|
||||
|
||||
enabled() {
|
||||
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
|
||||
1|true|yes|on) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
|
||||
sonar_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/sonarqube-quality-gate.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${sonar_status}" in
|
||||
ok|pass|passed|success) ;;
|
||||
*)
|
||||
echo "sonarqube gate failed: ${sonar_status}" >&2
|
||||
fail=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
|
||||
semgrep_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/semgrep-report.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
status = str(payload.get("status") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${semgrep_status}" in
|
||||
ok|pass|passed|success) ;;
|
||||
*)
|
||||
echo "semgrep gate failed: ${semgrep_status}" >&2
|
||||
fail=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
|
||||
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
|
||||
ironbank_required=1
|
||||
fi
|
||||
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
|
||||
supply_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/ironbank-compliance.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
compliant = payload.get("compliant")
|
||||
if compliant is True:
|
||||
print("ok")
|
||||
elif compliant is False:
|
||||
print("failed")
|
||||
else:
|
||||
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${supply_status}" in
|
||||
ok|pass|passed|success|compliant) ;;
|
||||
not_applicable|na|n/a)
|
||||
if enabled "${ironbank_required}"; then
|
||||
echo "supply chain gate required but status=${supply_status}" >&2
|
||||
fail=1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if enabled "${ironbank_required}"; then
|
||||
echo "supply chain gate failed: ${supply_status}" >&2
|
||||
fail=1
|
||||
else
|
||||
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
exit "${fail}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Resolve Flux branch') {
|
||||
steps {
|
||||
script {
|
||||
env.FLUX_BRANCH = sh(
|
||||
returnStdout: true,
|
||||
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
|
||||
).trim()
|
||||
if (!env.FLUX_BRANCH) {
|
||||
error('Flux branch not found in gotk-sync.yaml')
|
||||
}
|
||||
echo "Flux branch: ${env.FLUX_BRANCH}"
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Promote') {
|
||||
when {
|
||||
expression {
|
||||
def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
|
||||
return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
|
||||
}
|
||||
}
|
||||
steps {
|
||||
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache git >/dev/null
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y git >/dev/null
|
||||
fi
|
||||
fi
|
||||
cd "${WORKSPACE:-$PWD}"
|
||||
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
echo "workspace is not a git checkout; skipping promote"
|
||||
exit 0
|
||||
fi
|
||||
set +x
|
||||
git config user.email "jenkins@bstein.dev"
|
||||
git config user.name "jenkins"
|
||||
git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/titan/atlas-iac.git
|
||||
git push origin HEAD:${FLUX_BRANCH}
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
try {
|
||||
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
|
||||
try {
|
||||
junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
|
||||
} catch (Throwable err) {
|
||||
echo "junit step unavailable: ${err.class.simpleName}"
|
||||
}
|
||||
}
|
||||
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
|
||||
} catch (Throwable err) {
|
||||
if (err.class.simpleName == 'MissingContextVariableException') {
|
||||
echo 'workspace unavailable; skipping post-build artifact collection'
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
105
Makefile
105
Makefile
@ -1,105 +0,0 @@
|
||||
# Makefile
|
||||
SHELL := /usr/bin/env bash
|
||||
|
||||
KUSTOMIZE ?= kustomize
|
||||
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
|
||||
TERRAFORM ?= terraform
|
||||
TRIVY ?= trivy
|
||||
BUILD_DIR ?= build
|
||||
ATLAS_FLUX_ROOT := clusters/atlas/flux-system
|
||||
AETHER_FLUX_ROOT := clusters/aether/flux-system
|
||||
AETHER_TF_DIR := terraform/aether
|
||||
FLUX_KUSTOMIZATION ?= flux-system
|
||||
FLUX_NAMESPACE ?= flux-system
|
||||
FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
|
||||
TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
|
||||
TRIVY_DB_FLAGS ?= --skip-db-update
|
||||
|
||||
.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
|
||||
|
||||
help:
|
||||
@printf '%s\n' \
|
||||
'Targets:' \
|
||||
' render Render Atlas, Aether, services, and infrastructure.' \
|
||||
' render-atlas Render the Atlas Flux root.' \
|
||||
' render-aether Render the Aether Flux root.' \
|
||||
' render-services Render every service kustomization.' \
|
||||
' render-infrastructure Render infrastructure kustomizations.' \
|
||||
' validate Run render checks plus the local quality gate.' \
|
||||
' dashboards Regenerate Grafana dashboard artifacts.' \
|
||||
' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \
|
||||
' test Run the local quality gate.' \
|
||||
' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
|
||||
' flux-inventory Print Flux Kustomization inventory.' \
|
||||
' trivy-scan Run the local Trivy filesystem scan.' \
|
||||
' security-report Build IronBank report from build/trivy-fs.json.' \
|
||||
' security Run Trivy scan and build the IronBank report.' \
|
||||
' aether-fmt Check Terraform formatting for Aether.' \
|
||||
' aether-init Initialize Aether Terraform providers.' \
|
||||
' aether-validate Validate Aether Terraform after init.' \
|
||||
' aether-plan Plan Aether Terraform from local variables/env.'
|
||||
|
||||
render: render-atlas render-aether render-services render-infrastructure
|
||||
|
||||
render-atlas:
|
||||
$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
|
||||
|
||||
render-aether:
|
||||
$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
|
||||
|
||||
render-services:
|
||||
@set -euo pipefail; \
|
||||
while IFS= read -r k; do \
|
||||
d="$${k%/kustomization.yaml}"; \
|
||||
printf 'render %s\n' "$$d"; \
|
||||
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
|
||||
done < <(find services -name kustomization.yaml | sort)
|
||||
|
||||
render-infrastructure:
|
||||
@set -euo pipefail; \
|
||||
while IFS= read -r k; do \
|
||||
d="$${k%/kustomization.yaml}"; \
|
||||
printf 'render %s\n' "$$d"; \
|
||||
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
|
||||
done < <(find infrastructure -name kustomization.yaml | sort)
|
||||
|
||||
validate: render test
|
||||
|
||||
dashboards:
|
||||
$(PYTHON) scripts/render/dashboards_render_atlas.py --build
|
||||
$(PYTHON) scripts/render/dashboards_render_logs.py --build
|
||||
$(PYTHON) scripts/render/logging_render_observability.py --build
|
||||
|
||||
knowledge:
|
||||
$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
|
||||
|
||||
test:
|
||||
$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
|
||||
|
||||
flux-diff:
|
||||
flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
|
||||
|
||||
flux-inventory:
|
||||
$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
|
||||
|
||||
trivy-scan:
|
||||
mkdir -p $(BUILD_DIR)
|
||||
$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
|
||||
|
||||
security-report:
|
||||
@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
|
||||
$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
|
||||
|
||||
security: trivy-scan security-report
|
||||
|
||||
aether-fmt:
|
||||
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
|
||||
|
||||
aether-init:
|
||||
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
|
||||
|
||||
aether-validate:
|
||||
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
|
||||
|
||||
aether-plan:
|
||||
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan
|
||||
18
README.md
18
README.md
@ -1,17 +1,3 @@
|
||||
# atlas-iac
|
||||
# titan-iac
|
||||
|
||||
Flux-managed Kubernetes desired-state config for `bstein.dev`.
|
||||
|
||||
Canonical source URL:
|
||||
- `ssh://git@scm.bstein.dev:2242/titan/atlas-iac.git`
|
||||
|
||||
## Scope
|
||||
|
||||
This repo contains cluster configuration consumed by Flux:
|
||||
- platform/infrastructure manifests
|
||||
- service manifests and kustomizations
|
||||
- operational scripts for render/reconcile workflows
|
||||
|
||||
## Apply model
|
||||
|
||||
I use Git + Flux as the source of truth and avoid manual in-cluster edits for durable changes.
|
||||
Flux-managed Kubernetes cluster for bstein.dev services.
|
||||
|
||||
@ -1,552 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-agent-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-08
|
||||
- titan-14
|
||||
- titan-18
|
||||
- titan-19
|
||||
- titan-22
|
||||
- titan-24
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: hardware
|
||||
operator: In
|
||||
values:
|
||||
- rpi5
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
# The agent image build expands the SQLite and web build layers in
|
||||
# its workspace. Reserve enough node-local space to keep it off the
|
||||
# 25 GiB workers, whose unrequested Kaniko workspace was evicted.
|
||||
ephemeral-storage: 32Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
ephemeral-storage: 40Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Full reviewed commit already contained by titan/atlas-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 150, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-agent
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-agent.destination
|
||||
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Validate reviewed release source') {
|
||||
steps {
|
||||
container('python') {
|
||||
sh '''
|
||||
set -eu
|
||||
# Install the pinned test deps fully offline from the reviewed,
|
||||
# in-repo wheelhouse (ci/vendor/hermes-agent-test-wheels). --no-index
|
||||
# forbids any network index, so this stage never resolves
|
||||
# pypi.org/files.pythonhosted.org and cannot fail on public-internet
|
||||
# DNS. The wheels match this pipeline's arm64 python:3.12 container
|
||||
# (PyYAML is the cp312 manylinux aarch64 build; the rest are
|
||||
# py3-none-any). Bump the wheelhouse when these pins change.
|
||||
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
||||
--no-index --find-links="${WORKSPACE}/ci/vendor/hermes-agent-test-wheels" \
|
||||
--target=/tmp/hermes-agent-release-test-deps \
|
||||
pytest==8.3.4 PyYAML==6.0.2
|
||||
PYTHONPATH=/tmp/hermes-agent-release-test-deps \
|
||||
python3 -m pytest -q \
|
||||
testing/tests/test_hermes_image_builder.py \
|
||||
testing/tests/test_hermes_image_builder_adversarial.py \
|
||||
testing/tests/test_hermes_image_builder_coverage.py \
|
||||
testing/tests/test_hermes_image_builder_fresh_review.py \
|
||||
testing/tests/test_hermes_oci_promote.py \
|
||||
testing/tests/test_hermes_multiarch_combine.py \
|
||||
testing/tests/test_hermes_image_automation.py
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Reject replay before publish') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
python3 ci/scripts/hermes_image_release.py assert-absent \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build arm64 leg without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
minimum_available_kib=16777216
|
||||
available_kib="$(/busybox/df -Pk / | /busybox/awk 'NR == 2 { print $4 }')"
|
||||
case "${available_kib}" in
|
||||
''|*[!0-9]*)
|
||||
echo "cannot determine Kaniko ephemeral-storage availability" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "${available_kib}" -lt "${minimum_available_kib}" ]; then
|
||||
echo "Kaniko requires at least 16 GiB of free ephemeral storage" >&2
|
||||
exit 1
|
||||
fi
|
||||
'''
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-agent.destination)-arm64"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
|
||||
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
/busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build amd64 leg without a daemon') {
|
||||
agent {
|
||||
kubernetes {
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
# titan-24 is an accelerator node (not a general worker) that co-hosts the
|
||||
# out-of-cluster Sui validator. Pin the disposable amd64 build to it by
|
||||
# hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
|
||||
# opened to general cluster scheduling. The toleration + tight caps below keep
|
||||
# this off the validator's back.
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
kubernetes.io/hostname: titan-24
|
||||
tolerations:
|
||||
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
|
||||
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
|
||||
# build lands, and rely on the tight resource caps below (not scheduling
|
||||
# priority) to keep the disposable build from starving the validator.
|
||||
- operator: Exists
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 384Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1500m"
|
||||
memory: 3Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
steps {
|
||||
// This amd64 leg runs on its own fresh pod (titan-24), so it must check
|
||||
// out the SCM itself before the reviewed-revision git boundary check —
|
||||
// otherwise `git rev-parse origin/main` fails with "not a git repository".
|
||||
checkout scm
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-agent
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-agent.destination
|
||||
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
||||
'''
|
||||
}
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-agent.destination)-amd64"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
|
||||
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
/busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
stash(
|
||||
name: 'hermes-agent-amd64-evidence',
|
||||
includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
|
||||
)
|
||||
}
|
||||
}
|
||||
stage('Combine multi-arch index') {
|
||||
steps {
|
||||
unstash 'hermes-agent-amd64-evidence'
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
python3 ci/scripts/hermes_multiarch_combine.py \
|
||||
--destination "${destination}" \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--arm64-digest-file build/hermes-agent-arm64.digest \
|
||||
--arm64-image-file build/hermes-agent-arm64.image \
|
||||
--amd64-digest-file build/hermes-agent-amd64.digest \
|
||||
--amd64-image-file build/hermes-agent-amd64.image \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image
|
||||
test -s build/hermes-agent.digest
|
||||
test -s build/hermes-agent.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
python3 ci/scripts/hermes_image_release.py render \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
test -s build/hermes-agent-release/hermes-image-update.patch
|
||||
test -s build/hermes-agent-release/hermes-agent-image.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Verify and archive release evidence') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
expected_files="$(printf '%s\n' \
|
||||
build/hermes-agent.destination \
|
||||
build/hermes-agent.digest \
|
||||
build/hermes-agent.image \
|
||||
build/hermes-agent.source-revision \
|
||||
build/hermes-agent-arm64.digest \
|
||||
build/hermes-agent-arm64.image \
|
||||
build/hermes-agent-amd64.digest \
|
||||
build/hermes-agent-amd64.image \
|
||||
build/hermes-agent-release/hermes-agent-image.json \
|
||||
build/hermes-agent-release/hermes-image-update.patch \
|
||||
build/hermes-agent-release/hermes-kustomization.yaml \
|
||||
| LC_ALL=C sort)"
|
||||
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
||||
test "${actual_files}" = "${expected_files}"
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
python3 ci/scripts/hermes_image_release.py verify-evidence \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--image-file build/hermes-agent.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
}
|
||||
stage('Publish Flux release tag') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
source_revision="$(cat build/hermes-agent.source-revision)"
|
||||
python3 ci/scripts/hermes_oci_promote.py \
|
||||
--destination "${destination}" \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,339 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-chat-router-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
values: [titan-20]
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 128Mi}
|
||||
limits: {cpu: 500m, memory: 512Mi}
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 64Mi}
|
||||
limits: {cpu: 250m, memory: 256Mi}
|
||||
- name: golang
|
||||
image: golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
env:
|
||||
- {name: GOCACHE, value: /tmp/go-cache}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 1Gi}
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
ephemeral-storage: 2Gi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
ephemeral-storage: 5Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Full reviewed commit that must be contained by titan/atlas-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES CHAT ROUTER to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-chat-router'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 45, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES CHAT ROUTER"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse HEAD)"
|
||||
test "${main_revision}" = "$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-chat-router
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-chat-router.destination
|
||||
printf '%s\n' "${actual_revision}" \
|
||||
> build/hermes-chat-router.source-revision
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Validate reviewed router source') {
|
||||
steps {
|
||||
container('python') {
|
||||
sh '''
|
||||
set -eu
|
||||
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
||||
--target=/tmp/hermes-chat-router-test-deps \
|
||||
pytest==8.3.4 PyYAML==6.0.2
|
||||
PYTHONPATH=/tmp/hermes-chat-router-test-deps \
|
||||
python3 -m pytest -q \
|
||||
testing/tests/test_hermes_chat_router_release.py \
|
||||
testing/tests/test_hermes_oci_promote.py \
|
||||
testing/tests/test_hermes_image_automation.py
|
||||
'''
|
||||
}
|
||||
container('golang') {
|
||||
sh '''
|
||||
set -eu
|
||||
cd services/hermes/router
|
||||
GO111MODULE=off CGO_ENABLED=0 go test ./...
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Reject replay before publish') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-chat-router.destination)"
|
||||
source_revision="$(cat build/hermes-chat-router.source-revision)"
|
||||
python3 ci/scripts/hermes_chat_router_release.py assert-absent \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build and publish without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-chat-router.destination)"
|
||||
source_revision="$(cat build/hermes-chat-router.source-revision)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-chat-router" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-chat-router.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-chat-router.image" \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
|
||||
--label="org.opencontainers.image.title=hermes-chat-router" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
/busybox/chmod 644 \
|
||||
build/hermes-chat-router.digest build/hermes-chat-router.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-chat-router.destination)"
|
||||
source_revision="$(cat build/hermes-chat-router.source-revision)"
|
||||
python3 ci/scripts/hermes_chat_router_release.py render \
|
||||
--digest-file build/hermes-chat-router.digest \
|
||||
--image-file build/hermes-chat-router.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--manifest services/hermes/chat-router.yaml \
|
||||
--output-dir build/hermes-chat-router-release
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Verify and archive release evidence') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
expected_files="$(printf '%s\n' \
|
||||
build/hermes-chat-router.destination \
|
||||
build/hermes-chat-router.digest \
|
||||
build/hermes-chat-router.image \
|
||||
build/hermes-chat-router.source-revision \
|
||||
build/hermes-chat-router-release/hermes-chat-router-deployment.yaml \
|
||||
build/hermes-chat-router-release/hermes-chat-router-image-update.patch \
|
||||
build/hermes-chat-router-release/hermes-chat-router-image.json \
|
||||
| LC_ALL=C sort)"
|
||||
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
||||
test "${actual_files}" = "${expected_files}"
|
||||
destination="$(cat build/hermes-chat-router.destination)"
|
||||
source_revision="$(cat build/hermes-chat-router.source-revision)"
|
||||
python3 ci/scripts/hermes_chat_router_release.py verify-evidence \
|
||||
--digest-file build/hermes-chat-router.digest \
|
||||
--image-file build/hermes-chat-router.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--manifest services/hermes/chat-router.yaml \
|
||||
--output-dir build/hermes-chat-router-release
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-chat-router.destination,build/hermes-chat-router.digest,build/hermes-chat-router.image,build/hermes-chat-router.source-revision,build/hermes-chat-router-release/hermes-chat-router-deployment.yaml,build/hermes-chat-router-release/hermes-chat-router-image-update.patch,build/hermes-chat-router-release/hermes-chat-router-image.json',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
}
|
||||
stage('Publish Flux release tag') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-chat-router.destination)"
|
||||
source_revision="$(cat build/hermes-chat-router.source-revision)"
|
||||
python3 ci/scripts/hermes_oci_promote.py \
|
||||
--destination "${destination}" \
|
||||
--digest-file build/hermes-chat-router.digest \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,236 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-voice-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
tolerations:
|
||||
# Jetson kubelet/network maintenance can briefly outlast Kubernetes' five
|
||||
# minute default. Keep the disposable build workspace intact long enough
|
||||
# for the node to recover instead of restarting a large image expansion.
|
||||
- key: node.kubernetes.io/not-ready
|
||||
operator: Exists
|
||||
effect: NoExecute
|
||||
tolerationSeconds: 600
|
||||
- key: node.kubernetes.io/unreachable
|
||||
operator: Exists
|
||||
effect: NoExecute
|
||||
tolerationSeconds: 600
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
# Voice images expand the large Jetson base and two Whisper
|
||||
# models. Keep that transient I/O on the roomy idle
|
||||
# accelerator, away from Longhorn and the live speech node.
|
||||
values: [titan-20]
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 256Mi}
|
||||
limits: {cpu: 500m, memory: 512Mi}
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 64Mi}
|
||||
limits: {cpu: 500m, memory: 512Mi}
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
# The pinned Jetson Whisper base contains gst-ptp-helper with a
|
||||
# security.capability xattr. Kaniko needs SETFCAP only while
|
||||
# unpacking that reviewed base; the pod remains non-privileged.
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID", "SETFCAP"]
|
||||
runAsUser: 0
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
resources:
|
||||
# The STT image snapshots two checksum-pinned Whisper models. A 1 GiB
|
||||
# request let the builder overpack titan-20 and a 4 GiB cgroup killed
|
||||
# Kaniko while it emitted the large model layer.
|
||||
requests: {cpu: 250m, memory: 2Gi, ephemeral-storage: 10Gi}
|
||||
limits: {cpu: "2", memory: 6Gi, ephemeral-storage: 20Gi}
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish the reviewed voice image to Harbor.')
|
||||
choice(name: 'IMAGE_COMPONENT', choices: ['stt', 'tts'], description: 'Private voice component to build.')
|
||||
string(name: 'EXPECTED_SOURCE_REVISION', defaultValue: '', description: 'Full reviewed commit contained by main.')
|
||||
string(name: 'CONFIRM_PUBLISH', defaultValue: '', description: 'Exact component-specific confirmation.')
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 150, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps { checkout scm }
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
case "${IMAGE_COMPONENT}" in
|
||||
stt) expected_confirmation='PUBLISH HERMES STT' ;;
|
||||
tts) expected_confirmation='PUBLISH HERMES TTS' ;;
|
||||
*) echo 'IMAGE_COMPONENT must be stt or tts' >&2; exit 2 ;;
|
||||
esac
|
||||
test "${CONFIRM_PUBLISH}" = "${expected_confirmation}"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'') echo 'EXPECTED_SOURCE_REVISION must be a lowercase full commit' >&2; exit 2 ;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse HEAD)"
|
||||
test "${main_revision}" = "$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
case "${BUILD_NUMBER}" in ''|0*|*[!0-9]*) exit 2 ;; esac
|
||||
image="registry.bstein.dev/bstein/hermes-jetson-${IMAGE_COMPONENT}"
|
||||
printf '%s\n' "${image}:git-${actual_revision}-build-${BUILD_NUMBER}" > build/hermes-voice.destination
|
||||
printf '%s\n' "${actual_revision}" > build/hermes-voice.source-revision
|
||||
printf '%s\n' "${IMAGE_COMPONENT}" > build/hermes-voice.component
|
||||
test -f "dockerfiles/Dockerfile.hermes-jetson-${IMAGE_COMPONENT}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Validate reviewed voice source') {
|
||||
steps {
|
||||
container('python') {
|
||||
sh '''
|
||||
set -eu
|
||||
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
||||
--target=/tmp/hermes-voice-test-deps pytest==8.3.4 PyYAML==6.0.2
|
||||
python3 -m py_compile \
|
||||
dockerfiles/hermes-jetson-stt-server.py \
|
||||
dockerfiles/hermes-jetson-tts-server.py \
|
||||
dockerfiles/hermes_jetson_tts_cues.py
|
||||
PYTHONPATH=/tmp/hermes-voice-test-deps python3 -m pytest -q \
|
||||
testing/tests/test_hermes_stt_streaming.py \
|
||||
testing/tests/test_hermes_stt_rolling_model.py \
|
||||
testing/tests/test_hermes_tts_language_routing.py \
|
||||
testing/tests/test_hermes_voice_language_routing.py \
|
||||
testing/tests/test_hermes_oci_promote.py \
|
||||
testing/tests/test_hermes_image_automation.py \
|
||||
testing/tests/test_hermes_voice_release.py
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build and publish without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
component="$(cat build/hermes-voice.component)"
|
||||
destination="$(cat build/hermes-voice.destination)"
|
||||
source_revision="$(cat build/hermes-voice.source-revision)"
|
||||
config_path=/kaniko/.docker/config.json
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--compressed-caching=false \
|
||||
--snapshot-mode=redo \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-jetson-${component}" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-voice.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-voice.image" \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
|
||||
--label="org.opencontainers.image.title=hermes-jetson-${component}" \
|
||||
--cleanup --push-retry=3
|
||||
/busybox/chmod 644 build/hermes-voice.digest build/hermes-voice.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Verify, archive, and publish Flux release') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-voice.destination)"
|
||||
source_revision="$(cat build/hermes-voice.source-revision)"
|
||||
component="$(cat build/hermes-voice.component)"
|
||||
digest="$(cat build/hermes-voice.digest)"
|
||||
image="$(cat build/hermes-voice.image)"
|
||||
test "${image}" = "${destination}@${digest}"
|
||||
python3 ci/scripts/hermes_oci_promote.py \
|
||||
--destination "${destination}" \
|
||||
--digest-file build/hermes-voice.digest \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
> build/hermes-voice.promotion.json
|
||||
python3 -c 'import json; data=json.load(open("build/hermes-voice.promotion.json")); assert data["result"] in {"published", "already-present"}'
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-voice.component,build/hermes-voice.destination,build/hermes-voice.digest,build/hermes-voice.image,build/hermes-voice.source-revision,build/hermes-voice.promotion.json',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,559 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-webui-image-builder
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
# Keep disposable Kaniko expansion off the astreae Longhorn
|
||||
# replica nodes. titan-20 is the roomy ARM image builder and
|
||||
# has local NVMe for this transient I/O.
|
||||
values:
|
||||
- titan-20
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
runAsNonRoot: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 10Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
ephemeral-storage: 20Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Full reviewed commit that must be contained by titan/atlas-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES WEBUI to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-webui'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 150, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse HEAD)"
|
||||
test "${main_revision}" = "$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-webui
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-webui.destination
|
||||
printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Validate reviewed WebUI source') {
|
||||
steps {
|
||||
container('python') {
|
||||
sh '''
|
||||
set -eu
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends ffmpeg nodejs
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
command -v ffmpeg >/dev/null
|
||||
command -v node >/dev/null
|
||||
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
||||
--target=/tmp/hermes-webui-release-test-deps \
|
||||
pytest==8.3.4 PyYAML==6.0.2
|
||||
PYTHONPATH=/tmp/hermes-webui-release-test-deps \
|
||||
python3 -m pytest -q \
|
||||
testing/tests/test_hermes_chat_quality.py \
|
||||
testing/tests/test_hermes_handsfree_stt.py \
|
||||
testing/tests/test_hermes_voice_instrument.py \
|
||||
testing/tests/test_hermes_voice_full_duplex.py \
|
||||
testing/tests/test_hermes_voice_route_preflight.py \
|
||||
testing/tests/test_hermes_voice_preflight_delivery.py \
|
||||
testing/tests/test_hermes_thinking_voice_cues.py \
|
||||
testing/tests/test_hermes_voice_language_routing.py \
|
||||
testing/tests/test_hermes_webui_brand.py \
|
||||
testing/tests/test_hermes_webui_release.py \
|
||||
testing/tests/test_hermes_webui_hux_bff.py \
|
||||
testing/tests/test_hermes_webui_hux_context.py \
|
||||
testing/tests/test_hermes_webui_hux_integration.py \
|
||||
testing/tests/test_hermes_webui_hux_backend_e2e.py \
|
||||
testing/tests/test_hermes_hux_ui_runtime_wave_a.py \
|
||||
testing/tests/test_hermes_hux_runtime_autonomy_privacy.py \
|
||||
testing/tests/test_hermes_hux_runtime_stop.py \
|
||||
testing/tests/test_hermes_hux_ui_runtime_wave_b.py \
|
||||
testing/tests/test_hermes_hux_runtime_wave_c.py \
|
||||
testing/tests/test_hermes_hux_runtime_plugin.py \
|
||||
testing/tests/test_hermes_hux_runtime_vendor_parity.py \
|
||||
testing/tests/test_hermes_hux_delivery.py \
|
||||
testing/tests/test_hermes_oci_promote.py \
|
||||
testing/tests/test_hermes_multiarch_combine.py \
|
||||
testing/tests/test_hermes_image_automation.py
|
||||
HUX_BACKEND_TESTS="$(find testing/tests -maxdepth 1 -type f \
|
||||
-name 'test_hermes_hux_*.py' \
|
||||
! -name '*_ui_*' \
|
||||
! -name '*runtime*' \
|
||||
! -name '*delivery*' \
|
||||
| sort)"
|
||||
test -n "${HUX_BACKEND_TESTS}"
|
||||
PYTHONPATH=/tmp/hermes-webui-release-test-deps \
|
||||
python3 -m pytest -q ${HUX_BACKEND_TESTS}
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Reject replay before publish') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-webui.destination)"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
python3 ci/scripts/hermes_webui_release.py assert-absent \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build arm64 leg without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-webui.destination)-arm64"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
|
||||
--destination="${destination}" \
|
||||
--build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-webui-arm64.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-arm64.image" \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
|
||||
--label="org.opencontainers.image.title=hermes-webui" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
/busybox/chmod 644 build/hermes-webui-arm64.digest build/hermes-webui-arm64.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build amd64 leg without a daemon') {
|
||||
agent {
|
||||
kubernetes {
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-webui-image-builder-amd64
|
||||
spec:
|
||||
serviceAccountName: hermes-image-builder
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
# titan-24 is an accelerator node (not a general worker) that co-hosts the
|
||||
# out-of-cluster Sui validator. Pin the disposable amd64 build to it by
|
||||
# hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
|
||||
# opened to general cluster scheduling. The toleration + tight caps below keep
|
||||
# this off the validator's back.
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: amd64
|
||||
kubernetes.io/hostname: titan-24
|
||||
tolerations:
|
||||
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
|
||||
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
|
||||
# build lands, and rely on the tight resource caps below (not scheduling
|
||||
# priority) to keep the disposable build from starving the validator.
|
||||
- operator: Exists
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 384Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
ephemeral-storage: 10Gi
|
||||
limits:
|
||||
cpu: "1500m"
|
||||
memory: 3Gi
|
||||
ephemeral-storage: 20Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
steps {
|
||||
// This amd64 leg runs on its own fresh pod (titan-24), so it must check
|
||||
// out the SCM itself before the reviewed-revision git boundary check —
|
||||
// otherwise `git rev-parse origin/main` fails with "not a git repository".
|
||||
checkout scm
|
||||
container('jnlp') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
main_revision="$(git rev-parse HEAD)"
|
||||
test "${main_revision}" = "$(git rev-parse origin/main)"
|
||||
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
||||
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-webui
|
||||
case "${BUILD_NUMBER}" in
|
||||
''|0*|*[!0-9]*)
|
||||
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' \
|
||||
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
||||
> build/hermes-webui.destination
|
||||
printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
|
||||
'''
|
||||
}
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-webui.destination)-amd64"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
umask 022
|
||||
/kaniko/executor \
|
||||
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
||||
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
|
||||
--destination="${destination}" \
|
||||
--build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-webui-amd64.digest" \
|
||||
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-amd64.image" \
|
||||
--label="org.opencontainers.image.revision=${source_revision}" \
|
||||
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
|
||||
--label="org.opencontainers.image.title=hermes-webui" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
/busybox/chmod 644 build/hermes-webui-amd64.digest build/hermes-webui-amd64.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
stash(
|
||||
name: 'hermes-webui-amd64-evidence',
|
||||
includes: 'build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image'
|
||||
)
|
||||
}
|
||||
}
|
||||
stage('Combine multi-arch index') {
|
||||
steps {
|
||||
unstash 'hermes-webui-amd64-evidence'
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-webui.destination)"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
python3 ci/scripts/hermes_multiarch_combine.py \
|
||||
--destination "${destination}" \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--arm64-digest-file build/hermes-webui-arm64.digest \
|
||||
--arm64-image-file build/hermes-webui-arm64.image \
|
||||
--amd64-digest-file build/hermes-webui-amd64.digest \
|
||||
--amd64-image-file build/hermes-webui-amd64.image \
|
||||
--digest-file build/hermes-webui.digest \
|
||||
--image-file build/hermes-webui.image
|
||||
test -s build/hermes-webui.digest
|
||||
test -s build/hermes-webui.image
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-webui.destination)"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
python3 ci/scripts/hermes_webui_release.py render \
|
||||
--digest-file build/hermes-webui.digest \
|
||||
--image-file build/hermes-webui.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--chat-manifest services/hermes/chat-statefulset.yaml \
|
||||
--dashboard-manifest services/hermes/deployment.yaml \
|
||||
--output-dir build/hermes-webui-release
|
||||
test -s build/hermes-webui-release/hermes-webui-image-update.patch
|
||||
test -s build/hermes-webui-release/hermes-webui-image.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Verify and archive release evidence') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
expected_files="$(printf '%s\n' \
|
||||
build/hermes-webui.destination \
|
||||
build/hermes-webui.digest \
|
||||
build/hermes-webui.image \
|
||||
build/hermes-webui.source-revision \
|
||||
build/hermes-webui-arm64.digest \
|
||||
build/hermes-webui-arm64.image \
|
||||
build/hermes-webui-amd64.digest \
|
||||
build/hermes-webui-amd64.image \
|
||||
build/hermes-webui-release/hermes-chat-statefulset.yaml \
|
||||
build/hermes-webui-release/hermes-dashboard-deployment.yaml \
|
||||
build/hermes-webui-release/hermes-webui-image.json \
|
||||
build/hermes-webui-release/hermes-webui-image-update.patch \
|
||||
| LC_ALL=C sort)"
|
||||
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
||||
test "${actual_files}" = "${expected_files}"
|
||||
destination="$(cat build/hermes-webui.destination)"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
python3 ci/scripts/hermes_webui_release.py verify-evidence \
|
||||
--digest-file build/hermes-webui.digest \
|
||||
--image-file build/hermes-webui.image \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}" \
|
||||
--destination "${destination}" \
|
||||
--chat-manifest services/hermes/chat-statefulset.yaml \
|
||||
--dashboard-manifest services/hermes/deployment.yaml \
|
||||
--output-dir build/hermes-webui-release
|
||||
'''
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-arm64.digest,build/hermes-webui-arm64.image,build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch',
|
||||
allowEmptyArchive: false,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
}
|
||||
stage('Publish Flux release tag') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
destination="$(cat build/hermes-webui.destination)"
|
||||
source_revision="$(cat build/hermes-webui.source-revision)"
|
||||
python3 ci/scripts/hermes_oci_promote.py \
|
||||
--destination "${destination}" \
|
||||
--digest-file build/hermes-webui.digest \
|
||||
--source-revision "${source_revision}" \
|
||||
--build-number "${BUILD_NUMBER}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,511 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-06
|
||||
- titan-11
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-13
|
||||
- titan-15
|
||||
- titan-17
|
||||
- titan-19
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
jenkins/jenkins-jenkins-agent: "true"
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
|
||||
resources:
|
||||
requests:
|
||||
cpu: "25m"
|
||||
memory: "256Mi"
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python:3.12-slim
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
- name: quality-tools
|
||||
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
- name: semgrep
|
||||
image: semgrep/semgrep:1.171.0
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
"""
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PIP_DISABLE_PIP_VERSION_CHECK = '1'
|
||||
PYTHONUNBUFFERED = '1'
|
||||
SUITE_NAME = 'titan_iac'
|
||||
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
|
||||
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
|
||||
SONARQUBE_PROJECT_KEY = 'titan_iac'
|
||||
SONARQUBE_TOKEN = credentials('sonarqube-token')
|
||||
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
|
||||
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
|
||||
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
|
||||
QUALITY_GATE_SEMGREP_ENFORCE = '0'
|
||||
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
|
||||
QUALITY_GATE_IRONBANK_ENFORCE = '1'
|
||||
QUALITY_GATE_IRONBANK_REQUIRED = '0'
|
||||
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
|
||||
}
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Install deps') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends git ca-certificates
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
fi
|
||||
pip install --no-cache-dir -r ci/requirements.txt
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Prepare local quality evidence') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
python3 -m testing.quality_gate --profile local --build-dir build
|
||||
local_quality_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect Semgrep evidence') {
|
||||
steps {
|
||||
container('semgrep') {
|
||||
sh '''#!/bin/sh
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
|
||||
semgrep_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
|
||||
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect SonarQube evidence') {
|
||||
steps {
|
||||
container('quality-tools') {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
args=(
|
||||
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
|
||||
"-Dsonar.login=${SONARQUBE_TOKEN}"
|
||||
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.sources=."
|
||||
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
|
||||
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
|
||||
)
|
||||
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
|
||||
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
|
||||
set +e
|
||||
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
|
||||
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
|
||||
token = os.getenv('SONARQUBE_TOKEN', '').strip()
|
||||
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
|
||||
|
||||
payload = {
|
||||
"status": "ERROR",
|
||||
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
|
||||
}
|
||||
if host and project_key:
|
||||
task_file = Path('.scannerwork/report-task.txt')
|
||||
task_id = ''
|
||||
if task_file.exists():
|
||||
for line in task_file.read_text(encoding='utf-8').splitlines():
|
||||
key, _, value = line.partition('=')
|
||||
if key == 'ceTaskId':
|
||||
task_id = value.strip()
|
||||
break
|
||||
if task_id:
|
||||
ce_query = urllib.parse.urlencode({"id": task_id})
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
|
||||
if token:
|
||||
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
||||
ce_request.add_header("Authorization", f"Basic {encoded}")
|
||||
try:
|
||||
with urllib.request.urlopen(ce_request, timeout=12) as response:
|
||||
ce_payload = json.loads(response.read().decode("utf-8"))
|
||||
except Exception:
|
||||
time.sleep(3)
|
||||
continue
|
||||
status = str(ce_payload.get("task", {}).get("status", "")).upper()
|
||||
if status in {"SUCCESS", "FAILED", "CANCELED"}:
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
query = urllib.parse.urlencode({"projectKey": project_key})
|
||||
request = urllib.request.Request(
|
||||
f"{host}/api/qualitygates/project_status?{query}",
|
||||
method="GET",
|
||||
)
|
||||
if token:
|
||||
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
||||
request.add_header("Authorization", f"Basic {encoded}")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=12) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
payload = {"status": "ERROR", "error": str(exc)}
|
||||
|
||||
with open(report_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, sort_keys=True)
|
||||
handle.write("\\n")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Collect IronBank evidence') {
|
||||
steps {
|
||||
container('quality-tools') {
|
||||
sh '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
set +e
|
||||
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-check-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
|
||||
trivy_rc=$?
|
||||
set -e
|
||||
if [ "${trivy_rc}" -ne 0 ]; then
|
||||
rm -f build/trivy-fs.json
|
||||
cat > build/ironbank-compliance.json <<EOF
|
||||
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy scan failed","trivy_rc":${trivy_rc}}
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -s build/trivy-fs.json ]; then
|
||||
cat > build/ironbank-compliance.json <<EOF
|
||||
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
'''
|
||||
}
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
if [ -s build/trivy-fs.json ]; then
|
||||
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
|
||||
exit 0
|
||||
fi
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
|
||||
if report_path.exists():
|
||||
raise SystemExit(0)
|
||||
|
||||
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
|
||||
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
|
||||
payload = {
|
||||
"status": status or "unknown",
|
||||
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
|
||||
}
|
||||
payload = {k: v for k, v in payload.items() if v is not None}
|
||||
if "status" not in payload:
|
||||
payload["status"] = "unknown"
|
||||
payload["note"] = (
|
||||
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
|
||||
"or write build/ironbank-compliance.json in image-building repos."
|
||||
)
|
||||
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Run quality gate') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
set +e
|
||||
python3 -m testing.quality_gate --profile jenkins --build-dir build
|
||||
quality_gate_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Publish test metrics') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
export JUNIT_GLOB='build/junit-*.xml'
|
||||
export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
|
||||
export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
|
||||
python3 ci/scripts/publish_test_metrics.py
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Enforce quality gate') {
|
||||
steps {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
|
||||
fail=0
|
||||
if [ "${gate_rc}" -ne 0 ]; then
|
||||
echo "quality gate failed with rc=${gate_rc}" >&2
|
||||
fail=1
|
||||
fi
|
||||
|
||||
enabled() {
|
||||
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
|
||||
1|true|yes|on) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
|
||||
sonar_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/sonarqube-quality-gate.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${sonar_status}" in
|
||||
ok|pass|passed|success) ;;
|
||||
*)
|
||||
echo "sonarqube gate failed: ${sonar_status}" >&2
|
||||
fail=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
|
||||
semgrep_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/semgrep-report.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
status = str(payload.get("status") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${semgrep_status}" in
|
||||
ok|pass|passed|success) ;;
|
||||
*)
|
||||
echo "semgrep gate failed: ${semgrep_status}" >&2
|
||||
fail=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
|
||||
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
|
||||
ironbank_required=1
|
||||
fi
|
||||
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
|
||||
supply_status="$(python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path = Path("build/ironbank-compliance.json")
|
||||
if not path.exists():
|
||||
print("missing")
|
||||
raise SystemExit(0)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
print("error")
|
||||
raise SystemExit(0)
|
||||
compliant = payload.get("compliant")
|
||||
if compliant is True:
|
||||
print("ok")
|
||||
elif compliant is False:
|
||||
print("failed")
|
||||
else:
|
||||
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
|
||||
print(status or "missing")
|
||||
PY
|
||||
)"
|
||||
case "${supply_status}" in
|
||||
ok|pass|passed|success|compliant) ;;
|
||||
not_applicable|na|n/a)
|
||||
if enabled "${ironbank_required}"; then
|
||||
echo "supply chain gate required but status=${supply_status}" >&2
|
||||
fail=1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if enabled "${ironbank_required}"; then
|
||||
echo "supply chain gate failed: ${supply_status}" >&2
|
||||
fail=1
|
||||
else
|
||||
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
exit "${fail}"
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Resolve Flux branch') {
|
||||
steps {
|
||||
script {
|
||||
env.FLUX_BRANCH = sh(
|
||||
returnStdout: true,
|
||||
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
|
||||
).trim()
|
||||
if (!env.FLUX_BRANCH) {
|
||||
error('Flux branch not found in gotk-sync.yaml')
|
||||
}
|
||||
echo "Flux branch: ${env.FLUX_BRANCH}"
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Promote') {
|
||||
when {
|
||||
expression {
|
||||
def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
|
||||
return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
|
||||
}
|
||||
}
|
||||
steps {
|
||||
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache git >/dev/null
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y git >/dev/null
|
||||
fi
|
||||
fi
|
||||
cd "${WORKSPACE:-$PWD}"
|
||||
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
echo "workspace is not a git checkout; skipping promote"
|
||||
exit 0
|
||||
fi
|
||||
set +x
|
||||
git config user.email "jenkins@bstein.dev"
|
||||
git config user.name "jenkins"
|
||||
git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/titan/atlas-iac.git
|
||||
git push origin HEAD:${FLUX_BRANCH}
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
try {
|
||||
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
|
||||
try {
|
||||
junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
|
||||
} catch (Throwable err) {
|
||||
echo "junit step unavailable: ${err.class.simpleName}"
|
||||
}
|
||||
}
|
||||
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
|
||||
} catch (Throwable err) {
|
||||
if (err.class.simpleName == 'MissingContextVariableException') {
|
||||
echo 'workspace unavailable; skipping post-build artifact collection'
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
pytest==8.3.4
|
||||
pytest-cov==6.0.0
|
||||
coverage==7.6.10
|
||||
kubernetes==30.1.0
|
||||
PyYAML==6.0.2
|
||||
requests==2.32.3
|
||||
ruff==0.8.4
|
||||
@ -1,479 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and render a reviewable Hermes chat-router image release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-chat-router"
|
||||
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
HARBOR_PROJECT = "bstein"
|
||||
HARBOR_REPOSITORY = "hermes-chat-router"
|
||||
IMMUTABLE_REPOSITORY_PATTERN = "hermes-chat-router"
|
||||
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/hermes-chat-router:"
|
||||
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
|
||||
)
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never forward registry credentials to a redirect target."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value only when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: str, source_revision: str, build_number: str
|
||||
) -> tuple[str, str]:
|
||||
"""Bind the unique build tag to one reviewed source and Jenkins build."""
|
||||
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = _validated(build_number, BUILD_PATTERN, "build number")
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match or match.groups() != (revision, build):
|
||||
raise ValueError("destination does not match the reviewed revision and build")
|
||||
return revision, build
|
||||
|
||||
|
||||
def validate_kaniko_evidence(
|
||||
*, digest_text: str, image_text: str, destination: str
|
||||
) -> str:
|
||||
"""Cross-check Kaniko's two independent output files."""
|
||||
digests = digest_text.splitlines()
|
||||
images = image_text.splitlines()
|
||||
if len(digests) != 1 or len(images) != 1:
|
||||
raise ValueError("Kaniko evidence must contain exactly one line per file")
|
||||
digest = _validated(digests[0], DIGEST_PATTERN, "image digest")
|
||||
if images[0].strip() != f"{destination}@{digest}":
|
||||
raise ValueError("Kaniko image evidence does not match destination and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Return same-origin registry responses without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _authorization(username: str, password: str) -> str:
|
||||
"""Build a Basic header without placing credentials in a URL."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
return f"Basic {token}"
|
||||
|
||||
|
||||
def _artifact_response(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one exact Harbor artifact by candidate tag."""
|
||||
if not DESTINATION_PATTERN.fullmatch(destination):
|
||||
raise ValueError("invalid destination")
|
||||
tag = urllib.parse.quote(destination.rsplit(":", 1)[1], safe="")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{tag}?with_immutable_status=true",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Authorization": _authorization(username, password),
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _rules_response(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Read the complete bounded Harbor immutable-rule page."""
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
|
||||
"?page=1&page_size=100",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Authorization": _authorization(username, password),
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def _normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Select only immutable-policy fields used by this lane."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{key: item.get(key) for key in ("kind", "decoration", "pattern")}
|
||||
for item in rule.get("tag_selectors") or []
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{key: item.get(key) for key in ("kind", "decoration", "pattern")}
|
||||
for item in (rule.get("scope_selectors") or {}).get("repository", [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verify_immutable_policy(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Fail closed unless one exact active router immutability rule exists."""
|
||||
status, body, headers = _rules_response(
|
||||
username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
|
||||
try:
|
||||
rules = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
|
||||
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
|
||||
raise RuntimeError("Harbor immutable rule list has an invalid shape")
|
||||
total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if total is None or not str(total).isdecimal() or int(total) != len(rules):
|
||||
raise RuntimeError("Harbor immutable rule page is incomplete")
|
||||
expected = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": IMMUTABLE_TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
matches = [
|
||||
value
|
||||
for value in map(_normalized_rule, rules)
|
||||
if value["tag_selectors"] == expected["tag_selectors"]
|
||||
and value["scope_selectors"] == expected["scope_selectors"]
|
||||
]
|
||||
if matches != [expected]:
|
||||
raise RuntimeError("Harbor router immutable build-tag policy is not exact")
|
||||
|
||||
|
||||
def assert_tag_absent(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Reject replay before Kaniko can target an already-used build tag."""
|
||||
status, _ = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status == 404:
|
||||
return
|
||||
if status == 200:
|
||||
raise RuntimeError("Harbor destination tag already exists")
|
||||
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
|
||||
|
||||
|
||||
def verify_registry_digest(
|
||||
destination: str,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Verify Harbor's digest, immutable tag, and persisted source label."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
status, body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
|
||||
try:
|
||||
artifact = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
|
||||
tag = destination.rsplit(":", 1)[1]
|
||||
matching = [
|
||||
item
|
||||
for item in artifact.get("tags") or []
|
||||
if isinstance(item, dict) and item.get("name") == tag
|
||||
]
|
||||
labels = ((artifact.get("extra_attrs") or {}).get("config") or {}).get("Labels")
|
||||
if artifact.get("digest") != digest:
|
||||
raise RuntimeError("Harbor digest does not match Kaniko evidence")
|
||||
if len(matching) != 1 or matching[0].get("immutable") is not True:
|
||||
raise RuntimeError("Harbor did not enforce the candidate tag as immutable")
|
||||
if not isinstance(labels, dict) or labels.get(
|
||||
"org.opencontainers.image.revision"
|
||||
) != revision:
|
||||
raise RuntimeError("Harbor OCI source-revision label does not match")
|
||||
|
||||
|
||||
def render_workload(source: str, digest: str) -> str:
|
||||
"""Replace the single exact router image while preserving the Flux marker."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
identity = "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes-chat-router\n"
|
||||
if not source.startswith("# services/hermes/chat-router.yaml\n" + identity):
|
||||
raise ValueError("Flux target identity changed")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
for index, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("image: "):
|
||||
continue
|
||||
value = stripped.removeprefix("image: ").split(" #", 1)[0]
|
||||
image, separator, current_digest = value.rpartition("@")
|
||||
if separator and re.fullmatch(
|
||||
rf"{re.escape(DEFAULT_IMAGE)}(?::[A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})?",
|
||||
image,
|
||||
):
|
||||
_validated(current_digest, DIGEST_PATTERN, "current Flux image digest")
|
||||
matches.append(index)
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"expected exactly one router image; found {len(matches)}")
|
||||
index = matches[0]
|
||||
indent = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
comment = ""
|
||||
if " #" in lines[index]:
|
||||
comment = " #" + lines[index].split(" #", 1)[1].rstrip("\n")
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
lines[index] = f"{indent}image: {DEFAULT_IMAGE}@{digest}{comment}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _metadata(
|
||||
digest: str, revision: str, build: str, destination: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"build_number": build,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"flux_targets": ["apps/Deployment/hermes/hermes-chat-router"],
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": revision,
|
||||
}
|
||||
|
||||
|
||||
def _expected_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
manifest: Path,
|
||||
) -> dict[str, str]:
|
||||
source = manifest.read_text(encoding="utf-8")
|
||||
rendered = render_workload(source, digest)
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile="a/services/hermes/chat-router.yaml",
|
||||
tofile="b/services/hermes/chat-router.yaml",
|
||||
)
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches the Flux target")
|
||||
metadata = _metadata(digest, source_revision, build_number, destination)
|
||||
return {
|
||||
"hermes-chat-router-deployment.yaml": rendered,
|
||||
"hermes-chat-router-image-update.patch": patch,
|
||||
"hermes-chat-router-image.json": json.dumps(
|
||||
metadata, indent=2, sort_keys=True
|
||||
)
|
||||
+ "\n",
|
||||
}
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
manifest: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Write deterministic, credential-free Flux handoff evidence."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
revision, build = validate_destination(destination, source_revision, build_number)
|
||||
expected = _expected_artifacts(
|
||||
digest=digest,
|
||||
source_revision=revision,
|
||||
build_number=build,
|
||||
destination=destination,
|
||||
manifest=manifest,
|
||||
)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name, content in expected.items():
|
||||
(output_dir / name).write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
manifest: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Recompute and compare every archived handoff byte."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
revision, build = validate_destination(destination, source_revision, build_number)
|
||||
expected = _expected_artifacts(
|
||||
digest=digest,
|
||||
source_revision=revision,
|
||||
build_number=build,
|
||||
destination=destination,
|
||||
manifest=manifest,
|
||||
)
|
||||
entries = list(output_dir.iterdir())
|
||||
if {entry.name for entry in entries} != set(expected) or not all(
|
||||
entry.is_file() and not entry.is_symlink() for entry in entries
|
||||
):
|
||||
raise ValueError("release output must contain exactly three evidence files")
|
||||
for name, content in expected.items():
|
||||
if (output_dir / name).read_text(encoding="utf-8") != content:
|
||||
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
|
||||
|
||||
|
||||
def _credentials() -> tuple[str, str]:
|
||||
username = os.environ.get("HARBOR_USER", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
return username, password
|
||||
|
||||
|
||||
def _add_common(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run one fail-closed candidate or evidence operation."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
absent = commands.add_parser("assert-absent")
|
||||
_add_common(absent)
|
||||
for name in ("render", "verify-evidence"):
|
||||
command = commands.add_parser(name)
|
||||
_add_common(command)
|
||||
command.add_argument("--digest-file", required=True, type=Path)
|
||||
command.add_argument("--image-file", required=True, type=Path)
|
||||
command.add_argument("--manifest", required=True, type=Path)
|
||||
command.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
validate_destination(args.destination, args.source_revision, args.build_number)
|
||||
if args.command == "verify-evidence":
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
validate_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
manifest=args.manifest,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
username, password = _credentials()
|
||||
if args.command == "assert-absent":
|
||||
verify_immutable_policy(username=username, password=password)
|
||||
assert_tag_absent(
|
||||
args.destination, username=username, password=password
|
||||
)
|
||||
return 0
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
verify_registry_digest(
|
||||
args.destination,
|
||||
digest,
|
||||
args.source_revision,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
manifest=args.manifest,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -1,502 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and render a reviewable Hermes agent image release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
|
||||
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
HARBOR_PROJECT = "bstein"
|
||||
HARBOR_REPOSITORY = "hermes-agent"
|
||||
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
|
||||
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/hermes-agent:"
|
||||
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
|
||||
)
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never send registry credentials to a redirect target."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: str, source_revision: str, build_number: str
|
||||
) -> tuple[str, str]:
|
||||
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
|
||||
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = _validated(build_number, BUILD_PATTERN, "build number")
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match or match.groups() != (revision, build):
|
||||
raise ValueError(
|
||||
"destination must bind the reviewed revision and unique Jenkins build"
|
||||
)
|
||||
return revision, build
|
||||
|
||||
|
||||
def validate_kaniko_evidence(
|
||||
*, digest_text: str, image_text: str, destination: str
|
||||
) -> str:
|
||||
"""Cross-check both independent Kaniko output files against the destination."""
|
||||
digest_lines = digest_text.splitlines()
|
||||
image_lines = image_text.splitlines()
|
||||
if len(digest_lines) != 1:
|
||||
raise ValueError("Kaniko digest evidence must contain exactly one line")
|
||||
if len(image_lines) != 1:
|
||||
raise ValueError("Kaniko image evidence must contain exactly one line")
|
||||
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
|
||||
if image_lines[0].strip() != f"{destination}@{digest}":
|
||||
raise ValueError("Kaniko image evidence does not match destination and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Make a registry request without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _artifact_response(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one exact Harbor artifact by tag with bounded response size."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination)
|
||||
if not match:
|
||||
raise ValueError("invalid destination")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
tag = destination.rsplit(":", 1)[1]
|
||||
encoded_tag = urllib.parse.quote(tag, safe="")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
|
||||
"?with_immutable_status=true",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _immutable_rules_response(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Read the project policy with the same least-privilege publish identity."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
|
||||
"?page=1&page_size=100",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def _require_complete_rule_page(
|
||||
rules: list[dict[str, Any]], headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Require proof that the bounded first page contains every rule."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
|
||||
if int(raw_total) != len(rules):
|
||||
raise RuntimeError("Harbor immutable rule list was truncated")
|
||||
|
||||
|
||||
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Select only fields that bind the server-side build-tag policy."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in rule.get("tag_selectors") or []
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in (rule.get("scope_selectors") or {}).get("repository", [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verify_immutable_policy(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Fail closed before build unless the exact Harbor rule is active."""
|
||||
status, body, headers = _immutable_rules_response(
|
||||
username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
|
||||
try:
|
||||
rules = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
|
||||
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
|
||||
raise RuntimeError("Harbor immutable rule list has an invalid shape")
|
||||
_require_complete_rule_page(rules, headers)
|
||||
expected = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": IMMUTABLE_TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
matches = [
|
||||
_normalized_immutable_rule(item)
|
||||
for item in rules
|
||||
if _normalized_immutable_rule(item)["tag_selectors"]
|
||||
== expected["tag_selectors"]
|
||||
and _normalized_immutable_rule(item)["scope_selectors"]
|
||||
== expected["scope_selectors"]
|
||||
]
|
||||
if matches != [expected]:
|
||||
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
|
||||
|
||||
|
||||
def assert_tag_absent(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Reject replay before Kaniko can push an already-used immutable identity."""
|
||||
status, _body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status == 404:
|
||||
return
|
||||
if status == 200:
|
||||
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
|
||||
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
|
||||
|
||||
|
||||
def verify_registry_digest(
|
||||
destination: str,
|
||||
digest: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
status, body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
|
||||
try:
|
||||
artifact = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
|
||||
harbor_digest = str(artifact.get("digest") or "").strip()
|
||||
if not DIGEST_PATTERN.fullmatch(harbor_digest):
|
||||
raise RuntimeError("Harbor response omitted a valid artifact digest")
|
||||
if harbor_digest != digest:
|
||||
raise RuntimeError("Harbor digest does not match Kaniko evidence")
|
||||
expected_tag = destination.rsplit(":", 1)[1]
|
||||
matching_tags = [
|
||||
item
|
||||
for item in artifact.get("tags") or []
|
||||
if isinstance(item, dict) and item.get("name") == expected_tag
|
||||
]
|
||||
if len(matching_tags) != 1:
|
||||
raise RuntimeError("Harbor artifact does not contain the expected tag")
|
||||
if matching_tags[0].get("immutable") is not True:
|
||||
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
|
||||
|
||||
|
||||
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
|
||||
"""Replace exactly one matching Kustomize image digest without reformatting."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
if line.strip() != f"- name: {image}":
|
||||
continue
|
||||
name_indent = len(line) - len(line.lstrip())
|
||||
for candidate_index in range(index + 1, len(lines)):
|
||||
candidate = lines[candidate_index]
|
||||
stripped = candidate.strip()
|
||||
candidate_indent = len(candidate) - len(candidate.lstrip())
|
||||
if stripped.startswith("- name:") and candidate_indent == name_indent:
|
||||
break
|
||||
if stripped.startswith("digest:") and candidate_indent > name_indent:
|
||||
matches.append(candidate_index)
|
||||
break
|
||||
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected exactly one digest for image {image!r}; found {len(matches)}"
|
||||
)
|
||||
|
||||
index = matches[0]
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
value = lines[index].strip().removeprefix("digest:").strip()
|
||||
_current_digest, separator, comment = value.partition(" #")
|
||||
suffix = f" #{comment}" if separator else ""
|
||||
lines[index] = f"{prefix}digest: {digest}{suffix}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, str]:
|
||||
"""Write a rendered manifest, patch, and credential-free release metadata."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches the Flux manifest")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
|
||||
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
(output_dir / "hermes-agent-image.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest_file: Path,
|
||||
image_file: Path,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Revalidate the exact successful-build evidence without rewriting it."""
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=digest_file.read_text(encoding="utf-8"),
|
||||
image_text=image_file.read_text(encoding="utf-8"),
|
||||
destination=destination,
|
||||
)
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
expected_names = {
|
||||
"hermes-agent-image.json",
|
||||
"hermes-image-update.patch",
|
||||
"hermes-kustomization.yaml",
|
||||
}
|
||||
entries = list(output_dir.iterdir())
|
||||
if {entry.name for entry in entries} != expected_names or not all(
|
||||
entry.is_file() and not entry.is_symlink() for entry in entries
|
||||
):
|
||||
raise ValueError("release output must contain exactly three evidence files")
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
metadata = {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
expected = {
|
||||
"hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True)
|
||||
+ "\n",
|
||||
"hermes-image-update.patch": patch,
|
||||
"hermes-kustomization.yaml": rendered,
|
||||
}
|
||||
for name, expected_text in expected.items():
|
||||
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
|
||||
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
|
||||
|
||||
|
||||
def _credentials() -> tuple[str, str]:
|
||||
"""Read the masked, runtime-only Jenkins credential environment."""
|
||||
username = os.environ.get("HARBOR_USER", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
return username, password
|
||||
|
||||
|
||||
def _common_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fail closed around the unique tag, then verify and render after push."""
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
absent = commands.add_parser("assert-absent")
|
||||
_common_arguments(absent)
|
||||
render = commands.add_parser("render")
|
||||
_common_arguments(render)
|
||||
render.add_argument("--digest-file", required=True, type=Path)
|
||||
render.add_argument("--image-file", required=True, type=Path)
|
||||
render.add_argument("--kustomization", required=True, type=Path)
|
||||
render.add_argument("--output-dir", required=True, type=Path)
|
||||
verify = commands.add_parser("verify-evidence")
|
||||
_common_arguments(verify)
|
||||
verify.add_argument("--digest-file", required=True, type=Path)
|
||||
verify.add_argument("--image-file", required=True, type=Path)
|
||||
verify.add_argument("--kustomization", required=True, type=Path)
|
||||
verify.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
validate_destination(args.destination, args.source_revision, args.build_number)
|
||||
if args.command == "verify-evidence":
|
||||
validate_release_artifacts(
|
||||
digest_file=args.digest_file,
|
||||
image_file=args.image_file,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
username, password = _credentials()
|
||||
if args.command == "assert-absent":
|
||||
verify_immutable_policy(username=username, password=password)
|
||||
assert_tag_absent(args.destination, username=username, password=password)
|
||||
return 0
|
||||
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
verify_registry_digest(
|
||||
args.destination, digest, username=username, password=password
|
||||
)
|
||||
write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -1,393 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves.
|
||||
|
||||
Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on
|
||||
titan-24) and pushes each under an arch-suffixed candidate tag
|
||||
``...-build-<N>-<arch>``. Kaniko cannot combine, so this step:
|
||||
|
||||
1. Independently re-reads each per-arch candidate manifest from the registry and
|
||||
binds it to the exact Kaniko digest evidence.
|
||||
2. Proves each leaf really is the architecture it claims by reading its image
|
||||
config (a swapped or cross-built leaf fails closed here).
|
||||
3. Builds a Docker manifest *list* (not an OCI index) from the two verified
|
||||
leaves -- Docker manifest lists are already inside the promotion allow-list,
|
||||
so this keeps the security surface of ``hermes_oci_promote.py`` unchanged.
|
||||
4. Refuses to overwrite an existing final tag, PUTs the list to the final
|
||||
``...-build-<N>`` tag, and re-reads it to confirm the registry resolved the
|
||||
exact index digest referencing exactly the two expected leaves.
|
||||
|
||||
The output ``--digest-file``/``--image-file`` deliberately use the SAME format
|
||||
the single-arch Kaniko step produced (``<digest>`` and ``<destination>@<digest>``
|
||||
for the final, arch-less tag). The whole downstream evidence chain --
|
||||
``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py``
|
||||
-- therefore promotes the multi-arch INDEX with no further change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
REGISTRY_ORIGIN = "https://registry.bstein.dev"
|
||||
# The final (arch-less) Flux-visible tag; identical contract to the promoter.
|
||||
# Both Hermes images that the multi-arch pipelines publish share the exact same
|
||||
# arch-less final-tag contract; the repository name is the only difference and is
|
||||
# captured here so the combiner stays fail-closed to just these two components.
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/(?P<component>hermes-agent|hermes-webui):"
|
||||
r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
|
||||
)
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
# A per-arch leaf must be a single-image manifest, never itself a list/index.
|
||||
LEAF_MANIFEST_TYPES = {
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
}
|
||||
IMAGE_CONFIG_TYPES = {
|
||||
"application/vnd.docker.container.image.v1+json",
|
||||
"application/vnd.oci.image.config.v1+json",
|
||||
}
|
||||
# Deterministic architecture order -> deterministic manifest-list bytes/digest.
|
||||
ARCHITECTURES = ("amd64", "arm64")
|
||||
MAX_MANIFEST_BYTES = 4 * 1024 * 1024
|
||||
MAX_CONFIG_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never forward registry credentials to another origin."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Return normal and HTTP error responses without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _status(response: Any) -> int:
|
||||
"""Normalize urllib response and HTTPError status fields."""
|
||||
return int(getattr(response, "status", getattr(response, "code", 0)))
|
||||
|
||||
|
||||
def _authorization(username: str, password: str) -> str:
|
||||
"""Build a Basic authorization value without placing it in a URL."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
return f"Basic {encoded}"
|
||||
|
||||
|
||||
def _manifest_url(component: str, reference: str) -> str:
|
||||
"""Return one same-origin, path-escaped Docker Registry manifest URL."""
|
||||
encoded = urllib.parse.quote(reference, safe="")
|
||||
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}"
|
||||
|
||||
|
||||
def _blob_url(component: str, digest: str) -> str:
|
||||
"""Return one same-origin, path-escaped Docker Registry blob URL."""
|
||||
encoded = urllib.parse.quote(digest, safe="")
|
||||
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}"
|
||||
|
||||
|
||||
def _read_evidence_pair(
|
||||
*, digest_text: str, image_text: str, per_arch_tag_ref: str
|
||||
) -> str:
|
||||
"""Cross-check both Kaniko output files for one arch against its tag."""
|
||||
digest_lines = digest_text.splitlines()
|
||||
image_lines = image_text.splitlines()
|
||||
if len(digest_lines) != 1:
|
||||
raise ValueError("Kaniko digest evidence must contain exactly one line")
|
||||
if len(image_lines) != 1:
|
||||
raise ValueError("Kaniko image evidence must contain exactly one line")
|
||||
digest = digest_lines[0].strip()
|
||||
if not DIGEST_PATTERN.fullmatch(digest):
|
||||
raise ValueError("invalid per-arch image digest")
|
||||
if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}":
|
||||
raise ValueError("per-arch image evidence does not match tag and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _verified_leaf(
|
||||
*,
|
||||
component: str,
|
||||
per_arch_tag: str,
|
||||
architecture: str,
|
||||
expected_digest: str,
|
||||
authorization: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Re-read one per-arch leaf and prove its digest, type, and architecture."""
|
||||
accept = ", ".join(sorted(LEAF_MANIFEST_TYPES))
|
||||
request = urllib.request.Request(
|
||||
_manifest_url(component, per_arch_tag),
|
||||
headers={"Accept": accept, "Authorization": authorization},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 30) as response:
|
||||
if _status(response) != 200:
|
||||
raise RuntimeError(
|
||||
f"{architecture} leaf manifest returned HTTP {_status(response)}"
|
||||
)
|
||||
body = response.read(MAX_MANIFEST_BYTES + 1)
|
||||
if len(body) > MAX_MANIFEST_BYTES:
|
||||
raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit")
|
||||
observed_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
|
||||
if observed_digest != expected_digest:
|
||||
raise RuntimeError(f"{architecture} leaf digest does not match build evidence")
|
||||
# Defence in depth: the digest header is registry-asserted; recompute it too.
|
||||
if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest:
|
||||
raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest")
|
||||
if content_type not in LEAF_MANIFEST_TYPES:
|
||||
raise RuntimeError(f"{architecture} leaf is not a single-image manifest")
|
||||
try:
|
||||
manifest = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc
|
||||
config = manifest.get("config")
|
||||
if not isinstance(config, dict):
|
||||
raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor")
|
||||
config_digest = str(config.get("digest") or "")
|
||||
config_type = str(config.get("mediaType") or "")
|
||||
if not DIGEST_PATTERN.fullmatch(config_digest):
|
||||
raise RuntimeError(f"{architecture} leaf config digest is invalid")
|
||||
if config_type not in IMAGE_CONFIG_TYPES:
|
||||
raise RuntimeError(f"{architecture} leaf config media type is unsupported")
|
||||
config_request = urllib.request.Request(
|
||||
_blob_url(component, config_digest),
|
||||
headers={"Accept": config_type, "Authorization": authorization},
|
||||
method="GET",
|
||||
)
|
||||
with opener(config_request, 30) as response:
|
||||
if _status(response) != 200:
|
||||
raise RuntimeError(
|
||||
f"{architecture} leaf config returned HTTP {_status(response)}"
|
||||
)
|
||||
config_body = response.read(MAX_CONFIG_BYTES + 1)
|
||||
if len(config_body) > MAX_CONFIG_BYTES:
|
||||
raise RuntimeError(f"{architecture} leaf config exceeded the size limit")
|
||||
if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest:
|
||||
raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest")
|
||||
try:
|
||||
config_json = json.loads(config_body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc
|
||||
if config_json.get("architecture") != architecture:
|
||||
raise RuntimeError(
|
||||
f"{architecture} leaf config reports architecture "
|
||||
f"{config_json.get('architecture')!r}"
|
||||
)
|
||||
if config_json.get("os") != "linux":
|
||||
raise RuntimeError(f"{architecture} leaf config reports a non-linux os")
|
||||
return {
|
||||
"mediaType": content_type,
|
||||
"size": len(body),
|
||||
"digest": expected_digest,
|
||||
"platform": {"architecture": architecture, "os": "linux"},
|
||||
}
|
||||
|
||||
|
||||
def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes:
|
||||
"""Serialize the manifest list deterministically for a stable index digest."""
|
||||
document = {
|
||||
"schemaVersion": 2,
|
||||
"mediaType": DOCKER_MANIFEST_LIST,
|
||||
"manifests": descriptors,
|
||||
}
|
||||
return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def combine_multiarch_index(
|
||||
*,
|
||||
destination: str,
|
||||
arch_digests: dict[str, str],
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> dict[str, str]:
|
||||
"""Verify both leaves, publish, and re-verify one multi-arch index tag."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match:
|
||||
raise ValueError("invalid multi-arch destination")
|
||||
if set(arch_digests) != set(ARCHITECTURES):
|
||||
raise ValueError("expected exactly the arm64 and amd64 per-arch digests")
|
||||
component = match.group("component")
|
||||
index_tag = destination.rsplit(":", 1)[1]
|
||||
authorization = _authorization(username, password)
|
||||
|
||||
descriptors = [
|
||||
_verified_leaf(
|
||||
component=component,
|
||||
per_arch_tag=f"{index_tag}-{architecture}",
|
||||
architecture=architecture,
|
||||
expected_digest=arch_digests[architecture],
|
||||
authorization=authorization,
|
||||
opener=opener,
|
||||
)
|
||||
for architecture in ARCHITECTURES
|
||||
]
|
||||
manifest_list = _manifest_list_bytes(descriptors)
|
||||
if len(manifest_list) > MAX_MANIFEST_BYTES:
|
||||
raise RuntimeError("assembled manifest list exceeded the size limit")
|
||||
index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}"
|
||||
|
||||
index_url = _manifest_url(component, index_tag)
|
||||
head_request = urllib.request.Request(
|
||||
index_url,
|
||||
headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
|
||||
method="HEAD",
|
||||
)
|
||||
with opener(head_request, 20) as response:
|
||||
head_status = _status(response)
|
||||
existing_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
if head_status == 200:
|
||||
if existing_digest != index_digest:
|
||||
raise RuntimeError("final tag already exists with another index digest")
|
||||
result = "already-present"
|
||||
elif head_status == 404:
|
||||
put_request = urllib.request.Request(
|
||||
index_url,
|
||||
data=manifest_list,
|
||||
headers={
|
||||
"Authorization": authorization,
|
||||
"Content-Type": DOCKER_MANIFEST_LIST,
|
||||
},
|
||||
method="PUT",
|
||||
)
|
||||
with opener(put_request, 30) as response:
|
||||
put_status = _status(response)
|
||||
put_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
if put_status not in {201, 202}:
|
||||
raise RuntimeError(f"index manifest returned HTTP {put_status}")
|
||||
if put_digest and put_digest != index_digest:
|
||||
raise RuntimeError("index manifest digest changed during publish")
|
||||
result = "published"
|
||||
else:
|
||||
raise RuntimeError(f"final tag preflight returned HTTP {head_status}")
|
||||
|
||||
verify_request = urllib.request.Request(
|
||||
index_url,
|
||||
headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
|
||||
method="GET",
|
||||
)
|
||||
with opener(verify_request, 30) as response:
|
||||
if _status(response) != 200:
|
||||
raise RuntimeError(f"index verification returned HTTP {_status(response)}")
|
||||
verify_body = response.read(MAX_MANIFEST_BYTES + 1)
|
||||
if len(verify_body) > MAX_MANIFEST_BYTES:
|
||||
raise RuntimeError("index verification exceeded the size limit")
|
||||
verify_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
|
||||
if verify_digest != index_digest:
|
||||
raise RuntimeError("registry resolved the final tag to another index digest")
|
||||
if verify_type != DOCKER_MANIFEST_LIST:
|
||||
raise RuntimeError("registry did not store a Docker manifest list")
|
||||
try:
|
||||
published = json.loads(verify_body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("registry returned invalid index JSON") from exc
|
||||
published_leaves = {
|
||||
(
|
||||
str((item or {}).get("platform", {}).get("architecture")),
|
||||
str((item or {}).get("digest")),
|
||||
)
|
||||
for item in published.get("manifests") or []
|
||||
}
|
||||
expected_leaves = {
|
||||
(architecture, arch_digests[architecture]) for architecture in ARCHITECTURES
|
||||
}
|
||||
if published_leaves != expected_leaves:
|
||||
raise RuntimeError("published index does not reference the exact two leaves")
|
||||
|
||||
return {
|
||||
"component": component,
|
||||
"index_digest": index_digest,
|
||||
"index_tag": index_tag,
|
||||
"result": result,
|
||||
**{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES},
|
||||
}
|
||||
|
||||
|
||||
def _load_arch_digest(
|
||||
*, destination: str, architecture: str, digest_file: Path, image_file: Path
|
||||
) -> str:
|
||||
"""Bind one arch's two Kaniko evidence files to its arch-suffixed tag."""
|
||||
per_arch_tag_ref = f"{destination}-{architecture}"
|
||||
return _read_evidence_pair(
|
||||
digest_text=digest_file.read_text(encoding="utf-8"),
|
||||
image_text=image_file.read_text(encoding="utf-8"),
|
||||
per_arch_tag_ref=per_arch_tag_ref,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Combine two verified per-arch leaves and emit index digest evidence."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--arm64-digest-file", required=True, type=Path)
|
||||
parser.add_argument("--arm64-image-file", required=True, type=Path)
|
||||
parser.add_argument("--amd64-digest-file", required=True, type=Path)
|
||||
parser.add_argument("--amd64-image-file", required=True, type=Path)
|
||||
parser.add_argument("--digest-file", required=True, type=Path)
|
||||
parser.add_argument("--image-file", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
match = DESTINATION_PATTERN.fullmatch(args.destination.strip())
|
||||
if not match:
|
||||
raise ValueError("invalid multi-arch destination")
|
||||
if match.group("revision") != args.source_revision.strip():
|
||||
raise ValueError("destination revision does not match evidence")
|
||||
if match.group("build") != args.build_number.strip():
|
||||
raise ValueError("destination build number does not match evidence")
|
||||
destination = args.destination.strip()
|
||||
arch_digests = {
|
||||
"arm64": _load_arch_digest(
|
||||
destination=destination,
|
||||
architecture="arm64",
|
||||
digest_file=args.arm64_digest_file,
|
||||
image_file=args.arm64_image_file,
|
||||
),
|
||||
"amd64": _load_arch_digest(
|
||||
destination=destination,
|
||||
architecture="amd64",
|
||||
digest_file=args.amd64_digest_file,
|
||||
image_file=args.amd64_image_file,
|
||||
),
|
||||
}
|
||||
result = combine_multiarch_index(
|
||||
destination=destination,
|
||||
arch_digests=arch_digests,
|
||||
username=os.environ.get("HARBOR_USER", ""),
|
||||
password=os.environ.get("HARBOR_PASSWORD", ""),
|
||||
)
|
||||
args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8")
|
||||
args.image_file.write_text(
|
||||
f"{destination}@{result['index_digest']}\n", encoding="utf-8"
|
||||
)
|
||||
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
|
||||
print(json.dumps({"error": str(exc)}, sort_keys=True))
|
||||
return 1
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -1,193 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish a validated Hermes candidate manifest under its Flux release tag."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
REGISTRY_ORIGIN = "https://registry.bstein.dev"
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/"
|
||||
r"(?P<component>hermes-(?:agent|webui|chat-router|jetson-(?:stt|tts))):"
|
||||
r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
|
||||
)
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
MANIFEST_TYPES = {
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
}
|
||||
MAX_MANIFEST_BYTES = 4 * 1024 * 1024
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never forward registry credentials to another origin."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Return normal and HTTP error responses without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _status(response: Any) -> int:
|
||||
"""Normalize urllib response and HTTPError status fields."""
|
||||
return int(getattr(response, "status", getattr(response, "code", 0)))
|
||||
|
||||
|
||||
def _authorization(username: str, password: str) -> str:
|
||||
"""Build a Basic authorization value without placing it in a URL."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
return f"Basic {encoded}"
|
||||
|
||||
|
||||
def _validated_release(
|
||||
destination: str,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
) -> tuple[str, str, str]:
|
||||
"""Bind the candidate and release tag to one reviewed source and build."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match:
|
||||
raise ValueError("invalid Hermes candidate destination")
|
||||
if match.group("revision") != source_revision.strip():
|
||||
raise ValueError("candidate source revision does not match evidence")
|
||||
if match.group("build") != build_number.strip():
|
||||
raise ValueError("candidate build number does not match evidence")
|
||||
normalized_digest = digest.strip()
|
||||
if not DIGEST_PATTERN.fullmatch(normalized_digest):
|
||||
raise ValueError("invalid candidate digest")
|
||||
candidate_tag = destination.rsplit(":", 1)[1]
|
||||
return match.group("component"), candidate_tag, normalized_digest
|
||||
|
||||
|
||||
def _manifest_url(component: str, tag: str) -> str:
|
||||
"""Return one same-origin, path-escaped Docker Registry manifest URL."""
|
||||
encoded_tag = urllib.parse.quote(tag, safe="")
|
||||
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded_tag}"
|
||||
|
||||
|
||||
def promote_candidate(
|
||||
*,
|
||||
destination: str,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> dict[str, str]:
|
||||
"""Copy an exact candidate manifest to the immutable ``-release`` tag."""
|
||||
component, candidate_tag, normalized_digest = _validated_release(
|
||||
destination, digest, source_revision, build_number
|
||||
)
|
||||
release_tag = f"{candidate_tag}-release"
|
||||
authorization = _authorization(username, password)
|
||||
accept = ", ".join(sorted(MANIFEST_TYPES))
|
||||
candidate_request = urllib.request.Request(
|
||||
_manifest_url(component, candidate_tag),
|
||||
headers={"Accept": accept, "Authorization": authorization},
|
||||
method="GET",
|
||||
)
|
||||
with opener(candidate_request, 30) as response:
|
||||
if _status(response) != 200:
|
||||
raise RuntimeError(f"candidate manifest returned HTTP {_status(response)}")
|
||||
manifest = response.read(MAX_MANIFEST_BYTES + 1)
|
||||
if len(manifest) > MAX_MANIFEST_BYTES:
|
||||
raise RuntimeError("candidate manifest exceeded the size limit")
|
||||
observed_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
content_type = response.headers.get("Content-Type", "").split(";", 1)[0]
|
||||
if observed_digest != normalized_digest:
|
||||
raise RuntimeError("candidate manifest digest does not match build evidence")
|
||||
if content_type not in MANIFEST_TYPES:
|
||||
raise RuntimeError("candidate manifest returned an unsupported content type")
|
||||
|
||||
release_url = _manifest_url(component, release_tag)
|
||||
head_request = urllib.request.Request(
|
||||
release_url,
|
||||
headers={"Accept": accept, "Authorization": authorization},
|
||||
method="HEAD",
|
||||
)
|
||||
with opener(head_request, 20) as response:
|
||||
head_status = _status(response)
|
||||
existing_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
if head_status == 200:
|
||||
if existing_digest != normalized_digest:
|
||||
raise RuntimeError("release tag already exists with another digest")
|
||||
result = "already-present"
|
||||
elif head_status == 404:
|
||||
put_request = urllib.request.Request(
|
||||
release_url,
|
||||
data=manifest,
|
||||
headers={
|
||||
"Authorization": authorization,
|
||||
"Content-Type": content_type,
|
||||
},
|
||||
method="PUT",
|
||||
)
|
||||
with opener(put_request, 30) as response:
|
||||
put_status = _status(response)
|
||||
promoted_digest = response.headers.get("Docker-Content-Digest", "")
|
||||
if put_status not in {201, 202}:
|
||||
raise RuntimeError(f"release manifest returned HTTP {put_status}")
|
||||
if promoted_digest and promoted_digest != normalized_digest:
|
||||
raise RuntimeError("release manifest digest changed during promotion")
|
||||
result = "published"
|
||||
else:
|
||||
raise RuntimeError(f"release tag preflight returned HTTP {head_status}")
|
||||
|
||||
return {
|
||||
"component": component,
|
||||
"digest": normalized_digest,
|
||||
"release_tag": release_tag,
|
||||
"result": result,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Promote one evidence-bound candidate and print credential-free metadata."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--digest-file", required=True, type=Path)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = promote_candidate(
|
||||
destination=args.destination,
|
||||
digest=args.digest_file.read_text(encoding="utf-8"),
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
username=os.environ.get("HARBOR_USER", ""),
|
||||
password=os.environ.get("HARBOR_PASSWORD", ""),
|
||||
)
|
||||
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
|
||||
print(json.dumps({"error": str(exc)}, sort_keys=True))
|
||||
return 1
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -1,273 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render and revalidate the two-workload Hermes WebUI Flux handoff."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-webui"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
|
||||
DESTINATION_PATTERN = re.compile(
|
||||
r"^registry\.bstein\.dev/bstein/hermes-webui:"
|
||||
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
|
||||
)
|
||||
|
||||
|
||||
def validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: str, source_revision: str, build_number: str
|
||||
) -> tuple[str, str]:
|
||||
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
|
||||
revision = validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = validated(build_number, BUILD_PATTERN, "build number")
|
||||
match = DESTINATION_PATTERN.fullmatch(destination.strip())
|
||||
if not match or match.groups() != (revision, build):
|
||||
raise ValueError(
|
||||
"destination must bind the reviewed revision and unique Jenkins build"
|
||||
)
|
||||
return revision, build
|
||||
|
||||
|
||||
def render_workload(
|
||||
source: str,
|
||||
digest: str,
|
||||
*,
|
||||
kind: str,
|
||||
name: str,
|
||||
image: str = DEFAULT_IMAGE,
|
||||
expected_images: int | tuple[int, ...] = 1,
|
||||
) -> str:
|
||||
"""Replace every expected WebUI consumer in one exact Flux workload."""
|
||||
digest = validated(digest, DIGEST_PATTERN, "image digest")
|
||||
identity = re.compile(
|
||||
rf"\A(?:#[^\n]*\n)*apiVersion: apps/v1\nkind: {re.escape(kind)}\n"
|
||||
rf"metadata:\n name: {re.escape(name)}\n"
|
||||
)
|
||||
if not identity.search(source):
|
||||
raise ValueError(f"Flux target identity changed: expected {kind}/{name}")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
suffixes: dict[int, str] = {}
|
||||
for index, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("image: "):
|
||||
continue
|
||||
value, separator, comment = stripped.removeprefix("image: ").partition(" #")
|
||||
current_image, at, current_digest = value.rpartition("@")
|
||||
if not at or not re.fullmatch(
|
||||
rf"{re.escape(image)}(?::[A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})?",
|
||||
current_image,
|
||||
):
|
||||
continue
|
||||
validated(current_digest, DIGEST_PATTERN, "current Flux image digest")
|
||||
matches.append(index)
|
||||
suffixes[index] = f" #{comment}" if separator else ""
|
||||
allowed = (expected_images,) if isinstance(expected_images, int) else expected_images
|
||||
if not allowed or any(count < 1 for count in allowed) or len(matches) not in allowed:
|
||||
raise ValueError(
|
||||
f"expected {allowed} {image!r} image(s) in {kind}/{name}; "
|
||||
f"found {len(matches)}"
|
||||
)
|
||||
for index in matches:
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
lines[index] = f"{prefix}image: {image}@{digest}{suffixes[index]}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def render_hux_build_metadata(
|
||||
source: str, digest: str, source_revision: str, build_number: str
|
||||
) -> str:
|
||||
"""Bind HUX metadata when the activated sidecar fields are present."""
|
||||
digest = validated(digest, DIGEST_PATTERN, "image digest")
|
||||
revision = validated(source_revision, REVISION_PATTERN, "source revision")
|
||||
build = validated(build_number, BUILD_PATTERN, "build number")
|
||||
replacements = {
|
||||
"HUX_IMAGE_TAG": f"git-{revision}-build-{build}-release",
|
||||
"HUX_IMAGE_DIGEST": digest,
|
||||
}
|
||||
rendered = source
|
||||
# Block style only: Flux setters cannot attach to values inside flow
|
||||
# mappings, so the manifest keeps these as two-line entries with the
|
||||
# marker comment on the value scalar. The renderer stays a belt on top
|
||||
# of the Flux :tag/:digest setters and binds the same values.
|
||||
present = {
|
||||
name: rendered.count(f"- name: {name}\n") for name in replacements
|
||||
}
|
||||
if set(present.values()) == {0}:
|
||||
return rendered
|
||||
if any(count != 1 for count in present.values()):
|
||||
raise ValueError(f"incomplete HUX build binding fields: {present}")
|
||||
for name, value in replacements.items():
|
||||
pattern = re.compile(
|
||||
rf"^(?P<head>(?P<indent>\s*)- name: {name}\n(?P=indent) value: )"
|
||||
rf"[^#\n]+(?P<suffix> #[^\n]*)?$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
rendered, count = pattern.subn(
|
||||
lambda match: (
|
||||
f"{match.group('head')}{value}{match.group('suffix') or ''}"
|
||||
),
|
||||
rendered,
|
||||
)
|
||||
if count != 1:
|
||||
raise ValueError(f"expected exactly one {name} HUX build binding; found {count}")
|
||||
return rendered
|
||||
|
||||
|
||||
def _targets(chat_manifest: Path, dashboard_manifest: Path):
|
||||
return (
|
||||
(
|
||||
chat_manifest,
|
||||
"StatefulSet",
|
||||
"hermes-chat-tenant",
|
||||
"hermes-chat-statefulset.yaml",
|
||||
(1, 2, 3),
|
||||
),
|
||||
(
|
||||
dashboard_manifest,
|
||||
"Deployment",
|
||||
"hermes",
|
||||
"hermes-dashboard-deployment.yaml",
|
||||
1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _metadata(
|
||||
digest: str, source_revision: str, build_number: str, destination: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"build_number": build_number,
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"flux_targets": [
|
||||
"apps/StatefulSet/hermes/hermes-chat-tenant",
|
||||
"apps/Deployment/hermes/hermes",
|
||||
],
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
|
||||
|
||||
def _rendered_and_patch(
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
chat_manifest: Path,
|
||||
dashboard_manifest: Path,
|
||||
) -> tuple[dict[str, str], str]:
|
||||
rendered_targets: dict[str, str] = {}
|
||||
patch_parts: list[str] = []
|
||||
for path, kind, name, artifact_name, expected_images in _targets(chat_manifest, dashboard_manifest):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
rendered = render_workload(
|
||||
source,
|
||||
digest,
|
||||
kind=kind,
|
||||
name=name,
|
||||
expected_images=expected_images,
|
||||
)
|
||||
if name == "hermes-chat-tenant":
|
||||
rendered = render_hux_build_metadata(
|
||||
rendered, digest, source_revision, build_number
|
||||
)
|
||||
rendered_targets[artifact_name] = rendered
|
||||
patch_parts.append(
|
||||
"".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{path.name}",
|
||||
tofile=f"b/services/hermes/{path.name}",
|
||||
)
|
||||
)
|
||||
)
|
||||
return rendered_targets, "".join(patch_parts)
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
chat_manifest: Path,
|
||||
dashboard_manifest: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Write two rendered Flux targets, one patch, and credential-free evidence."""
|
||||
digest = validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
rendered_targets, patch = _rendered_and_patch(
|
||||
digest, source_revision, build_number, chat_manifest, dashboard_manifest
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches every Flux target")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for artifact_name, rendered in rendered_targets.items():
|
||||
(output_dir / artifact_name).write_text(rendered, encoding="utf-8")
|
||||
(output_dir / "hermes-webui-image-update.patch").write_text(patch, encoding="utf-8")
|
||||
metadata = _metadata(digest, source_revision, build_number, destination)
|
||||
(output_dir / "hermes-webui-image.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
chat_manifest: Path,
|
||||
dashboard_manifest: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Revalidate the exact successful-build evidence without rewriting it."""
|
||||
digest = validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision, build_number = validate_destination(
|
||||
destination, source_revision, build_number
|
||||
)
|
||||
expected_names = {
|
||||
"hermes-webui-image.json",
|
||||
"hermes-webui-image-update.patch",
|
||||
"hermes-chat-statefulset.yaml",
|
||||
"hermes-dashboard-deployment.yaml",
|
||||
}
|
||||
entries = list(output_dir.iterdir())
|
||||
if {entry.name for entry in entries} != expected_names or not all(
|
||||
entry.is_file() and not entry.is_symlink() for entry in entries
|
||||
):
|
||||
raise ValueError("release output must contain exactly four evidence files")
|
||||
rendered_targets, patch = _rendered_and_patch(
|
||||
digest, source_revision, build_number, chat_manifest, dashboard_manifest
|
||||
)
|
||||
metadata = _metadata(digest, source_revision, build_number, destination)
|
||||
expected = {
|
||||
"hermes-webui-image.json": json.dumps(metadata, indent=2, sort_keys=True)
|
||||
+ "\n",
|
||||
"hermes-webui-image-update.patch": patch,
|
||||
**rendered_targets,
|
||||
}
|
||||
for name, expected_text in expected.items():
|
||||
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
|
||||
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
|
||||
@ -1,469 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and render a reviewable Hermes WebUI image release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from hermes_webui_flux_release import (
|
||||
DEFAULT_IMAGE as DEFAULT_IMAGE,
|
||||
DESTINATION_PATTERN,
|
||||
DIGEST_PATTERN,
|
||||
REVISION_PATTERN,
|
||||
render_hux_build_metadata as render_hux_build_metadata,
|
||||
render_workload as render_workload,
|
||||
validate_destination,
|
||||
validate_release_artifacts as validate_flux_release_artifacts,
|
||||
validated as _validated,
|
||||
write_release_artifacts,
|
||||
)
|
||||
|
||||
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
|
||||
HARBOR_PROJECT = "bstein"
|
||||
HARBOR_REPOSITORY = "hermes-webui"
|
||||
IMMUTABLE_REPOSITORY_PATTERN = "hermes-webui"
|
||||
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Never send registry credentials to a redirect target."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def validate_kaniko_evidence(
|
||||
*, digest_text: str, image_text: str, destination: str
|
||||
) -> str:
|
||||
"""Cross-check both independent Kaniko output files against the destination."""
|
||||
digest_lines = digest_text.splitlines()
|
||||
image_lines = image_text.splitlines()
|
||||
if len(digest_lines) != 1:
|
||||
raise ValueError("Kaniko digest evidence must contain exactly one line")
|
||||
if len(image_lines) != 1:
|
||||
raise ValueError("Kaniko image evidence must contain exactly one line")
|
||||
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
|
||||
if image_lines[0].strip() != f"{destination}@{digest}":
|
||||
raise ValueError("Kaniko image evidence does not match destination and digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
|
||||
"""Make a registry request without following redirects."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _artifact_response(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one exact Harbor artifact by tag with bounded response size."""
|
||||
match = DESTINATION_PATTERN.fullmatch(destination)
|
||||
if not match:
|
||||
raise ValueError("invalid destination")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
tag = destination.rsplit(":", 1)[1]
|
||||
encoded_tag = urllib.parse.quote(tag, safe="")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
|
||||
"?with_immutable_status=true",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _immutable_rules_response(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes, dict[str, str]]:
|
||||
"""Read the project policy with the same least-privilege publish identity."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
|
||||
"?page=1&page_size=100",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
|
||||
return int(response.status), body, dict(response.headers)
|
||||
|
||||
|
||||
def _require_complete_rule_page(
|
||||
rules: list[dict[str, Any]], headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Require proof that the bounded first page contains every rule."""
|
||||
raw_total = next(
|
||||
(value for key, value in headers.items() if key.lower() == "x-total-count"),
|
||||
None,
|
||||
)
|
||||
if raw_total is None or not str(raw_total).isdecimal():
|
||||
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
|
||||
if int(raw_total) != len(rules):
|
||||
raise RuntimeError("Harbor immutable rule list was truncated")
|
||||
|
||||
|
||||
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Select only fields that bind the server-side build-tag policy."""
|
||||
return {
|
||||
"disabled": bool(rule.get("disabled", False)),
|
||||
"action": rule.get("action"),
|
||||
"template": rule.get("template"),
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in rule.get("tag_selectors") or []
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": item.get("kind"),
|
||||
"decoration": item.get("decoration"),
|
||||
"pattern": item.get("pattern"),
|
||||
}
|
||||
for item in (rule.get("scope_selectors") or {}).get("repository", [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verify_immutable_policy(
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Fail closed before build unless the exact Harbor rule is active."""
|
||||
status, body, headers = _immutable_rules_response(
|
||||
username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
|
||||
try:
|
||||
rules = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
|
||||
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
|
||||
raise RuntimeError("Harbor immutable rule list has an invalid shape")
|
||||
_require_complete_rule_page(rules, headers)
|
||||
expected = {
|
||||
"disabled": False,
|
||||
"action": "immutable",
|
||||
"template": "immutable_template",
|
||||
"tag_selectors": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "matches",
|
||||
"pattern": IMMUTABLE_TAG_PATTERN,
|
||||
}
|
||||
],
|
||||
"scope_selectors": {
|
||||
"repository": [
|
||||
{
|
||||
"kind": "doublestar",
|
||||
"decoration": "repoMatches",
|
||||
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
matches = [
|
||||
_normalized_immutable_rule(item)
|
||||
for item in rules
|
||||
if _normalized_immutable_rule(item)["tag_selectors"]
|
||||
== expected["tag_selectors"]
|
||||
and _normalized_immutable_rule(item)["scope_selectors"]
|
||||
== expected["scope_selectors"]
|
||||
]
|
||||
if matches != [expected]:
|
||||
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
|
||||
|
||||
|
||||
def assert_tag_absent(
|
||||
destination: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Reject replay before Kaniko can push an already-used immutable identity."""
|
||||
status, _body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status == 404:
|
||||
return
|
||||
if status == 200:
|
||||
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
|
||||
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
|
||||
|
||||
|
||||
OCI_REVISION_LABEL = "org.opencontainers.image.revision"
|
||||
|
||||
|
||||
def _child_artifact_response(
|
||||
child_digest: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> tuple[int, bytes]:
|
||||
"""Read one per-arch child artifact of a multi-arch index by its digest."""
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are empty")
|
||||
encoded = urllib.parse.quote(child_digest, safe="")
|
||||
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
request = urllib.request.Request(
|
||||
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
|
||||
f"{HARBOR_REPOSITORY}/artifacts/{encoded}"
|
||||
"?with_immutable_status=true",
|
||||
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
|
||||
method="GET",
|
||||
)
|
||||
with opener(request, 20) as response:
|
||||
body = response.read(1_048_577)
|
||||
if len(body) > 1_048_576:
|
||||
raise RuntimeError("Harbor child artifact response exceeded the size limit")
|
||||
return int(response.status), body
|
||||
|
||||
|
||||
def _config_labels(artifact: dict[str, Any]) -> Any:
|
||||
"""Extract the OCI config labels Harbor reports for one artifact, if any."""
|
||||
return ((artifact.get("extra_attrs") or {}).get("config") or {}).get("Labels")
|
||||
|
||||
|
||||
def _verify_source_revision_label(
|
||||
artifact: dict[str, Any],
|
||||
source_revision: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any],
|
||||
) -> None:
|
||||
"""Assert the published image carries the reviewed source revision label.
|
||||
|
||||
A single-arch image exposes ``org.opencontainers.image.revision`` on its own
|
||||
config. A multi-arch manifest list has no top-level config, so Harbor reports
|
||||
the label on each per-arch child instead; verify every child in that case.
|
||||
"""
|
||||
labels = _config_labels(artifact)
|
||||
if isinstance(labels, dict):
|
||||
if labels.get(OCI_REVISION_LABEL) != source_revision:
|
||||
raise RuntimeError("Harbor OCI source-revision label does not match")
|
||||
return
|
||||
references = artifact.get("references")
|
||||
if not isinstance(references, list) or not references:
|
||||
raise RuntimeError("Harbor artifact omitted OCI image labels")
|
||||
for reference in references:
|
||||
child_digest = str((reference or {}).get("child_digest") or "").strip()
|
||||
if not DIGEST_PATTERN.fullmatch(child_digest):
|
||||
raise RuntimeError("Harbor index reference omitted a valid child digest")
|
||||
status, body = _child_artifact_response(
|
||||
child_digest, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(
|
||||
f"Harbor child artifact verification returned HTTP {status}"
|
||||
)
|
||||
try:
|
||||
child = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid child artifact JSON") from exc
|
||||
child_labels = _config_labels(child)
|
||||
if not isinstance(child_labels, dict):
|
||||
raise RuntimeError("Harbor artifact omitted OCI image labels")
|
||||
if child_labels.get(OCI_REVISION_LABEL) != source_revision:
|
||||
raise RuntimeError("Harbor OCI source-revision label does not match")
|
||||
|
||||
|
||||
def verify_registry_digest(
|
||||
destination: str,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
*,
|
||||
username: str,
|
||||
password: str,
|
||||
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
|
||||
) -> None:
|
||||
"""Verify Harbor resolves the tag, digest, and persisted source revision."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision = _validated(
|
||||
source_revision, REVISION_PATTERN, "source revision"
|
||||
)
|
||||
status, body = _artifact_response(
|
||||
destination, username=username, password=password, opener=opener
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
|
||||
try:
|
||||
artifact = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
|
||||
harbor_digest = str(artifact.get("digest") or "").strip()
|
||||
if not DIGEST_PATTERN.fullmatch(harbor_digest):
|
||||
raise RuntimeError("Harbor response omitted a valid artifact digest")
|
||||
if harbor_digest != digest:
|
||||
raise RuntimeError("Harbor digest does not match Kaniko evidence")
|
||||
expected_tag = destination.rsplit(":", 1)[1]
|
||||
matching_tags = [
|
||||
item
|
||||
for item in artifact.get("tags") or []
|
||||
if isinstance(item, dict) and item.get("name") == expected_tag
|
||||
]
|
||||
if len(matching_tags) != 1:
|
||||
raise RuntimeError("Harbor artifact does not contain the expected tag")
|
||||
if matching_tags[0].get("immutable") is not True:
|
||||
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
|
||||
_verify_source_revision_label(
|
||||
artifact,
|
||||
source_revision,
|
||||
username=username,
|
||||
password=password,
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
|
||||
def validate_release_artifacts(
|
||||
*,
|
||||
digest_file: Path,
|
||||
image_file: Path,
|
||||
source_revision: str,
|
||||
build_number: str,
|
||||
destination: str,
|
||||
chat_manifest: Path,
|
||||
dashboard_manifest: Path,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Revalidate the exact successful-build evidence without rewriting it."""
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=digest_file.read_text(encoding="utf-8"),
|
||||
image_text=image_file.read_text(encoding="utf-8"),
|
||||
destination=destination,
|
||||
)
|
||||
validate_flux_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=source_revision,
|
||||
build_number=build_number,
|
||||
destination=destination,
|
||||
chat_manifest=chat_manifest,
|
||||
dashboard_manifest=dashboard_manifest,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
|
||||
def _credentials() -> tuple[str, str]:
|
||||
"""Read the masked, runtime-only Jenkins credential environment."""
|
||||
username = os.environ.get("HARBOR_USER", "")
|
||||
password = os.environ.get("HARBOR_PASSWORD", "")
|
||||
if not username or not password:
|
||||
raise RuntimeError("Harbor credentials are unavailable")
|
||||
return username, password
|
||||
|
||||
|
||||
def _common_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--build-number", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fail closed around the unique tag, then verify and render after push."""
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
absent = commands.add_parser("assert-absent")
|
||||
_common_arguments(absent)
|
||||
render = commands.add_parser("render")
|
||||
_common_arguments(render)
|
||||
render.add_argument("--digest-file", required=True, type=Path)
|
||||
render.add_argument("--image-file", required=True, type=Path)
|
||||
render.add_argument("--chat-manifest", required=True, type=Path)
|
||||
render.add_argument("--dashboard-manifest", required=True, type=Path)
|
||||
render.add_argument("--output-dir", required=True, type=Path)
|
||||
verify = commands.add_parser("verify-evidence")
|
||||
_common_arguments(verify)
|
||||
verify.add_argument("--digest-file", required=True, type=Path)
|
||||
verify.add_argument("--image-file", required=True, type=Path)
|
||||
verify.add_argument("--chat-manifest", required=True, type=Path)
|
||||
verify.add_argument("--dashboard-manifest", required=True, type=Path)
|
||||
verify.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
validate_destination(args.destination, args.source_revision, args.build_number)
|
||||
if args.command == "verify-evidence":
|
||||
validate_release_artifacts(
|
||||
digest_file=args.digest_file,
|
||||
image_file=args.image_file,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
chat_manifest=args.chat_manifest,
|
||||
dashboard_manifest=args.dashboard_manifest,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
username, password = _credentials()
|
||||
if args.command == "assert-absent":
|
||||
verify_immutable_policy(username=username, password=password)
|
||||
assert_tag_absent(args.destination, username=username, password=password)
|
||||
return 0
|
||||
|
||||
digest = validate_kaniko_evidence(
|
||||
digest_text=args.digest_file.read_text(encoding="utf-8"),
|
||||
image_text=args.image_file.read_text(encoding="utf-8"),
|
||||
destination=args.destination,
|
||||
)
|
||||
verify_registry_digest(
|
||||
args.destination,
|
||||
digest,
|
||||
args.source_revision,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=args.source_revision,
|
||||
build_number=args.build_number,
|
||||
destination=args.destination,
|
||||
chat_manifest=args.chat_manifest,
|
||||
dashboard_manifest=args.dashboard_manifest,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -1,379 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish titan-iac quality-gate results to Pushgateway."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from glob import glob
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from ci.scripts import publish_test_metrics_quality as _quality_helpers
|
||||
|
||||
CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
|
||||
_build_check_statuses = _quality_helpers._build_check_statuses
|
||||
_combine_statuses = _quality_helpers._combine_statuses
|
||||
_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
|
||||
_infer_semgrep_status = _quality_helpers._infer_semgrep_status
|
||||
_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
|
||||
_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
|
||||
_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
|
||||
_load_optional_json = _quality_helpers._load_optional_json
|
||||
_normalize_result_status = _quality_helpers._normalize_result_status
|
||||
|
||||
TEST_CASE_LABEL_MAX_BYTES = 240
|
||||
|
||||
|
||||
def _escape_label(value: str) -> str:
|
||||
"""Escape a Prometheus label value without changing its content."""
|
||||
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
|
||||
|
||||
|
||||
def _label_str(labels: dict[str, str]) -> str:
|
||||
"""Render a stable Prometheus label set from a mapping."""
|
||||
parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val]
|
||||
return "{" + ",".join(parts) + "}" if parts else ""
|
||||
|
||||
|
||||
def _bounded_test_name(value: str) -> str:
|
||||
"""Keep test labels readable, unique, and safely below scraper limits."""
|
||||
encoded = value.encode("utf-8")
|
||||
if len(encoded) <= TEST_CASE_LABEL_MAX_BYTES:
|
||||
return value
|
||||
digest = hashlib.sha256(encoded).hexdigest()[:16]
|
||||
suffix = f"...[sha256:{digest}]"
|
||||
prefix = encoded[: TEST_CASE_LABEL_MAX_BYTES - len(suffix)]
|
||||
return prefix.decode("utf-8", errors="ignore") + suffix
|
||||
|
||||
|
||||
def _read_text(url: str) -> str:
|
||||
"""Fetch a plain-text response body from the given URL."""
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return response.read().decode("utf-8")
|
||||
|
||||
|
||||
def _post_text(url: str, payload: str) -> None:
|
||||
"""PUT a plain-text payload and fail on any 4xx/5xx response."""
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=payload.encode("utf-8"),
|
||||
method="PUT",
|
||||
headers={"Content-Type": "text/plain"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
if response.status >= 400:
|
||||
raise RuntimeError(f"push failed with status={response.status}")
|
||||
|
||||
|
||||
def _parse_junit(path: str) -> dict[str, int]:
|
||||
"""Parse a JUnit XML file into aggregate test counters."""
|
||||
if not os.path.exists(path):
|
||||
return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
|
||||
|
||||
tree = ET.parse(path)
|
||||
root = tree.getroot()
|
||||
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
|
||||
|
||||
suites: list[ET.Element]
|
||||
if root.tag == "testsuite":
|
||||
suites = [root]
|
||||
elif root.tag == "testsuites":
|
||||
suites = [elem for elem in root if elem.tag == "testsuite"]
|
||||
else:
|
||||
suites = []
|
||||
|
||||
for suite in suites:
|
||||
for key in totals:
|
||||
raw_value = suite.attrib.get(key, "0")
|
||||
try:
|
||||
totals[key] += int(float(raw_value))
|
||||
except ValueError:
|
||||
totals[key] += 0
|
||||
return totals
|
||||
|
||||
|
||||
def _collect_junit_totals(pattern: str) -> dict[str, int]:
|
||||
"""Sum JUnit counters across every XML file matching the pattern."""
|
||||
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
|
||||
for path in sorted(glob(pattern)):
|
||||
parsed = _parse_junit(path)
|
||||
for key in totals:
|
||||
totals[key] += parsed[key]
|
||||
return totals
|
||||
|
||||
|
||||
def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
|
||||
"""Collect individual JUnit test-case statuses for flaky-test trend panels."""
|
||||
cases: list[tuple[str, str]] = []
|
||||
for path in sorted(glob(pattern)):
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
root = ET.parse(path).getroot()
|
||||
suites: list[ET.Element]
|
||||
if root.tag == "testsuite":
|
||||
suites = [root]
|
||||
elif root.tag == "testsuites":
|
||||
suites = [elem for elem in root if elem.tag == "testsuite"]
|
||||
else:
|
||||
suites = []
|
||||
for suite in suites:
|
||||
for test_case in suite.findall("testcase"):
|
||||
case_name = test_case.attrib.get("name", "").strip()
|
||||
class_name = test_case.attrib.get("classname", "").strip()
|
||||
if not case_name:
|
||||
continue
|
||||
full_name = f"{class_name}.{case_name}" if class_name else case_name
|
||||
status = "passed"
|
||||
if test_case.find("failure") is not None or test_case.find("error") is not None:
|
||||
status = "failed"
|
||||
elif test_case.find("skipped") is not None:
|
||||
status = "skipped"
|
||||
cases.append((_bounded_test_name(full_name), status))
|
||||
return cases
|
||||
|
||||
|
||||
def _read_exit_code(path: str) -> int:
|
||||
"""Read the quality-gate exit code, defaulting to failure if missing."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return int(handle.read().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 1
|
||||
|
||||
|
||||
def _load_summary(path: str) -> dict:
|
||||
"""Load the JSON quality-gate summary, returning an empty mapping on error."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _summary_float(summary: dict, key: str) -> float:
|
||||
"""Extract a float-like value from the summary, defaulting to 0.0."""
|
||||
value = summary.get(key)
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _summary_int(summary: dict, key: str) -> int:
|
||||
"""Extract an int-like value from the summary, defaulting to 0."""
|
||||
value = summary.get(key)
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
return 0
|
||||
|
||||
|
||||
def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float:
|
||||
"""Return the current counter value for a labeled metric if present."""
|
||||
text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics")
|
||||
for line in text.splitlines():
|
||||
if not line.startswith(metric + "{"):
|
||||
continue
|
||||
if any(f'{key}="{value}"' not in line for key, value in labels.items()):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
try:
|
||||
return float(parts[1])
|
||||
except ValueError:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def _build_payload(
|
||||
suite: str,
|
||||
status: str,
|
||||
tests: dict[str, int],
|
||||
test_cases: list[tuple[str, str]],
|
||||
ok_count: int,
|
||||
failed_count: int,
|
||||
branch: str,
|
||||
build_number: str,
|
||||
jenkins_job: str,
|
||||
summary: dict | None = None,
|
||||
workspace_line_coverage_percent: float = 0.0,
|
||||
source_files_total: int = 0,
|
||||
source_lines_over_500: int = 0,
|
||||
check_statuses: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Build the Pushgateway payload for the current suite run."""
|
||||
passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0)
|
||||
build_labels = _label_str(
|
||||
{
|
||||
"suite": suite,
|
||||
"branch": branch or "unknown",
|
||||
"build_number": build_number or "unknown",
|
||||
"jenkins_job": jenkins_job or suite,
|
||||
}
|
||||
)
|
||||
test_case_base_labels = {
|
||||
"suite": suite,
|
||||
"branch": branch or "unknown",
|
||||
"build_number": build_number or "unknown",
|
||||
"jenkins_job": jenkins_job or suite,
|
||||
}
|
||||
lines = [
|
||||
"# TYPE platform_quality_gate_runs_total counter",
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}',
|
||||
"# TYPE titan_iac_quality_gate_tests_total gauge",
|
||||
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}',
|
||||
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests["failures"]}',
|
||||
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests["errors"]}',
|
||||
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests["skipped"]}',
|
||||
"# TYPE titan_iac_quality_gate_run_status gauge",
|
||||
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}',
|
||||
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}',
|
||||
"# TYPE platform_quality_gate_build_info gauge",
|
||||
f"platform_quality_gate_build_info{build_labels} 1",
|
||||
"# TYPE titan_iac_quality_gate_build_info gauge",
|
||||
f"titan_iac_quality_gate_build_info{build_labels} 1",
|
||||
"# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
|
||||
f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}',
|
||||
"# TYPE platform_quality_gate_source_files_total gauge",
|
||||
f'platform_quality_gate_source_files_total{{suite="{suite}"}} {source_files_total}',
|
||||
"# TYPE platform_quality_gate_source_lines_over_500_total gauge",
|
||||
f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
|
||||
]
|
||||
if check_statuses:
|
||||
lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
|
||||
for check_name in CANONICAL_CHECKS:
|
||||
check_status = check_statuses.get(check_name, "not_applicable")
|
||||
lines.append(
|
||||
f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1'
|
||||
)
|
||||
lines.append("# TYPE platform_quality_gate_test_case_result gauge")
|
||||
if test_cases:
|
||||
for test_name, test_status in test_cases:
|
||||
labels = {
|
||||
**test_case_base_labels,
|
||||
"test": _bounded_test_name(test_name),
|
||||
"status": test_status,
|
||||
}
|
||||
lines.append(
|
||||
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
|
||||
)
|
||||
else:
|
||||
labels = {**test_case_base_labels, "test": "__no_test_cases__", "status": "skipped"}
|
||||
lines.append(
|
||||
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Publish the quality-gate metrics and print a compact run summary."""
|
||||
suite = os.getenv("SUITE_NAME", "titan_iac")
|
||||
pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
|
||||
job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
|
||||
junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
|
||||
exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc"))
|
||||
summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json")
|
||||
branch = os.getenv("BRANCH_NAME") or os.getenv("GIT_BRANCH") or "unknown"
|
||||
if branch.startswith("origin/"):
|
||||
branch = branch[len("origin/") :]
|
||||
build_number = os.getenv("BUILD_NUMBER", "")
|
||||
jenkins_job = os.getenv("JOB_NAME", "titan-iac")
|
||||
|
||||
tests = _collect_junit_totals(junit_glob)
|
||||
test_cases = _collect_junit_cases(junit_glob)
|
||||
exit_code = _read_exit_code(exit_code_path)
|
||||
status = "ok" if exit_code == 0 else "failed"
|
||||
summary = _load_summary(summary_path)
|
||||
workspace_line_coverage_percent = _summary_float(summary, "workspace_line_coverage_percent")
|
||||
if workspace_line_coverage_percent <= 0:
|
||||
workspace_line_coverage_percent = _infer_workspace_coverage_percent(summary, "build/coverage-unit.xml")
|
||||
source_files_total = _summary_int(summary, "source_files_total")
|
||||
source_lines_over_500 = _summary_int(summary, "source_lines_over_500")
|
||||
if source_lines_over_500 <= 0:
|
||||
source_lines_over_500 = _infer_source_lines_over_500(summary)
|
||||
sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
|
||||
semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
|
||||
supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
|
||||
truthy = {"1", "true", "yes", "on"}
|
||||
supply_chain_required = (
|
||||
os.getenv("QUALITY_GATE_IRONBANK_REQUIRED", "0").strip().lower() in truthy
|
||||
or os.getenv("PUBLISH_IMAGES", "false").strip().lower() in truthy
|
||||
)
|
||||
check_statuses = _build_check_statuses(
|
||||
summary=summary,
|
||||
tests=tests,
|
||||
workspace_line_coverage_percent=workspace_line_coverage_percent,
|
||||
source_lines_over_500=source_lines_over_500,
|
||||
sonarqube_report=sonarqube_report,
|
||||
semgrep_report=semgrep_report,
|
||||
supply_chain_report=supply_chain_report,
|
||||
supply_chain_required=supply_chain_required,
|
||||
)
|
||||
|
||||
ok_count = int(
|
||||
_fetch_existing_counter(
|
||||
pushgateway_url,
|
||||
"platform_quality_gate_runs_total",
|
||||
{"job": job_name, "suite": suite, "status": "ok"},
|
||||
)
|
||||
)
|
||||
failed_count = int(
|
||||
_fetch_existing_counter(
|
||||
pushgateway_url,
|
||||
"platform_quality_gate_runs_total",
|
||||
{"job": job_name, "suite": suite, "status": "failed"},
|
||||
)
|
||||
)
|
||||
if status == "ok":
|
||||
ok_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
payload = _build_payload(
|
||||
suite=suite,
|
||||
status=status,
|
||||
tests=tests,
|
||||
test_cases=test_cases,
|
||||
ok_count=ok_count,
|
||||
failed_count=failed_count,
|
||||
branch=branch,
|
||||
build_number=build_number,
|
||||
jenkins_job=jenkins_job,
|
||||
summary=summary,
|
||||
workspace_line_coverage_percent=workspace_line_coverage_percent,
|
||||
source_files_total=source_files_total,
|
||||
source_lines_over_500=source_lines_over_500,
|
||||
check_statuses=check_statuses,
|
||||
)
|
||||
push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}"
|
||||
_post_text(push_url, payload)
|
||||
|
||||
summary = {
|
||||
"suite": suite,
|
||||
"status": status,
|
||||
"tests_total": tests["tests"],
|
||||
"tests_failed": tests["failures"],
|
||||
"tests_error": tests["errors"],
|
||||
"tests_skipped": tests["skipped"],
|
||||
"ok_count": ok_count,
|
||||
"failed_count": failed_count,
|
||||
"checks_recorded": len(check_statuses),
|
||||
"workspace_line_coverage_percent": workspace_line_coverage_percent,
|
||||
"source_files_total": source_files_total,
|
||||
"source_lines_over_500": source_lines_over_500,
|
||||
}
|
||||
print(json.dumps(summary, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@ -1,221 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quality/status helpers for publish_test_metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
SUCCESS_STATUSES = {"ok", "pass", "passed", "success", "compliant"}
|
||||
NOT_APPLICABLE_STATUSES = {"not_applicable", "n/a", "na", "none", "skipped"}
|
||||
FAILED_STATUSES = {"failed", "fail", "error", "errors", "warn", "warning", "red"}
|
||||
|
||||
CANONICAL_CHECKS = [
|
||||
"tests",
|
||||
"coverage",
|
||||
"loc",
|
||||
"docs_naming",
|
||||
"gate_glue",
|
||||
"sonarqube",
|
||||
"semgrep",
|
||||
"supply_chain",
|
||||
]
|
||||
|
||||
|
||||
def _infer_workspace_coverage_percent(summary: dict, default_xml: str) -> float:
|
||||
"""Infer workspace line coverage from quality summary coverage XML metadata."""
|
||||
results = summary.get("results", []) if isinstance(summary, dict) else []
|
||||
coverage_xml = default_xml
|
||||
for result in results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
if str(result.get("name") or "").strip().lower() != "coverage":
|
||||
continue
|
||||
candidate = str(result.get("coverage_xml") or "").strip()
|
||||
if candidate:
|
||||
coverage_xml = candidate
|
||||
break
|
||||
xml_path = Path(coverage_xml)
|
||||
if not xml_path.exists():
|
||||
return 0.0
|
||||
try:
|
||||
root = ET.parse(xml_path).getroot()
|
||||
line_rate = root.attrib.get("line-rate")
|
||||
if line_rate is None:
|
||||
return 0.0
|
||||
return float(line_rate) * 100.0
|
||||
except (ET.ParseError, OSError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _infer_source_lines_over_500(summary: dict) -> int:
|
||||
"""Infer over-limit source file count from hygiene issue payloads."""
|
||||
results = summary.get("results", []) if isinstance(summary, dict) else []
|
||||
for result in results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
if str(result.get("name") or "").strip().lower() not in {"hygiene", "loc", "smell"}:
|
||||
continue
|
||||
issues = result.get("issues")
|
||||
if not isinstance(issues, list):
|
||||
continue
|
||||
return sum(1 for item in issues if isinstance(item, str) and item.startswith("file exceeds"))
|
||||
return 0
|
||||
|
||||
|
||||
def _normalize_result_status(value: str | None, default: str = "failed") -> str:
|
||||
"""Map arbitrary check status text into canonical check result buckets."""
|
||||
if not value:
|
||||
return default
|
||||
normalized = value.strip().lower()
|
||||
if normalized in SUCCESS_STATUSES:
|
||||
return "ok"
|
||||
if normalized in NOT_APPLICABLE_STATUSES:
|
||||
return "not_applicable"
|
||||
if normalized in FAILED_STATUSES:
|
||||
return "failed"
|
||||
return default
|
||||
|
||||
|
||||
def _load_optional_json(path: str | None) -> dict:
|
||||
"""Load an optional JSON report file, returning an empty object when absent."""
|
||||
if not path:
|
||||
return {}
|
||||
candidate = Path(path)
|
||||
if not candidate.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(candidate.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
def _combine_statuses(statuses: list[str]) -> str:
|
||||
"""Roll up many check statuses into one canonical result."""
|
||||
if not statuses:
|
||||
return "not_applicable"
|
||||
if any(status == "failed" for status in statuses):
|
||||
return "failed"
|
||||
if all(status == "not_applicable" for status in statuses):
|
||||
return "not_applicable"
|
||||
if all(status in {"ok", "not_applicable"} for status in statuses):
|
||||
return "ok"
|
||||
return "failed"
|
||||
|
||||
|
||||
def _infer_sonarqube_status(report: dict) -> str:
|
||||
"""Infer canonical SonarQube check status from its JSON report payload."""
|
||||
if not report:
|
||||
return "not_applicable"
|
||||
status = (
|
||||
report.get("projectStatus", {}).get("status")
|
||||
or report.get("qualityGate", {}).get("status")
|
||||
or report.get("status")
|
||||
)
|
||||
return _normalize_result_status(str(status) if status is not None else None, default="failed")
|
||||
|
||||
|
||||
def _infer_supply_chain_status(report: dict, required: bool) -> str:
|
||||
"""Infer canonical supply-chain status from IronBank/artifact report payload."""
|
||||
if not report:
|
||||
return "failed" if required else "not_applicable"
|
||||
compliant = report.get("compliant")
|
||||
if isinstance(compliant, bool):
|
||||
if compliant:
|
||||
return "ok"
|
||||
return "failed" if required else "not_applicable"
|
||||
status = report.get("status")
|
||||
if status is None:
|
||||
return "failed" if required else "not_applicable"
|
||||
normalized = _normalize_result_status(str(status), default="failed")
|
||||
if normalized == "failed" and not required:
|
||||
return "not_applicable"
|
||||
if normalized == "not_applicable" and required:
|
||||
return "failed"
|
||||
return normalized
|
||||
|
||||
|
||||
def _infer_semgrep_status(report: dict) -> str:
|
||||
"""Infer canonical Semgrep check status from its JSON report payload."""
|
||||
if not report:
|
||||
return "not_applicable"
|
||||
status = report.get("status")
|
||||
if status is None:
|
||||
blocking_findings = report.get("blocking_findings")
|
||||
errors_total = report.get("errors_total")
|
||||
if isinstance(blocking_findings, int) and isinstance(errors_total, int):
|
||||
return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
|
||||
return _normalize_result_status(str(status) if status is not None else None, default="failed")
|
||||
|
||||
|
||||
def _build_check_statuses(
|
||||
summary: dict | None,
|
||||
tests: dict[str, int],
|
||||
workspace_line_coverage_percent: float,
|
||||
source_lines_over_500: int,
|
||||
sonarqube_report: dict,
|
||||
semgrep_report: dict,
|
||||
supply_chain_report: dict,
|
||||
supply_chain_required: bool,
|
||||
) -> dict[str, str]:
|
||||
"""Generate the canonical quality-check status map for dashboarding."""
|
||||
raw_results = summary.get("results", []) if isinstance(summary, dict) else []
|
||||
status_by_name: dict[str, str] = {}
|
||||
for result in raw_results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
check_name = str(result.get("name") or "").strip().lower()
|
||||
if not check_name:
|
||||
continue
|
||||
status_by_name[check_name] = _normalize_result_status(result.get("status"), default="failed")
|
||||
|
||||
tests_status = status_by_name.get("tests")
|
||||
if not tests_status:
|
||||
candidate_keys = ["unit", "integration", "e2e", "pytest", "test", "tests"]
|
||||
candidates = [status_by_name[key] for key in candidate_keys if key in status_by_name]
|
||||
if candidates:
|
||||
tests_status = _combine_statuses(candidates)
|
||||
elif tests["tests"] > 0:
|
||||
tests_status = "ok" if (tests["failures"] + tests["errors"]) == 0 else "failed"
|
||||
else:
|
||||
tests_status = "not_applicable"
|
||||
|
||||
coverage_status = status_by_name.get("coverage")
|
||||
if not coverage_status:
|
||||
if workspace_line_coverage_percent > 0:
|
||||
coverage_status = "ok" if workspace_line_coverage_percent >= 95.0 else "failed"
|
||||
else:
|
||||
coverage_status = "not_applicable"
|
||||
|
||||
loc_status = status_by_name.get("loc")
|
||||
if not loc_status:
|
||||
loc_status = "ok" if source_lines_over_500 == 0 else "failed"
|
||||
|
||||
docs_naming_status = status_by_name.get("docs_naming")
|
||||
if not docs_naming_status:
|
||||
candidates = [status_by_name[key] for key in ["docs", "hygiene", "smell", "lint", "naming"] if key in status_by_name]
|
||||
docs_naming_status = _combine_statuses(candidates) if candidates else "not_applicable"
|
||||
|
||||
gate_glue_status = status_by_name.get("gate_glue")
|
||||
if not gate_glue_status:
|
||||
candidates = [status_by_name[key] for key in ["gate_glue", "glue", "gate"] if key in status_by_name]
|
||||
gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
|
||||
|
||||
sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
|
||||
semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
|
||||
supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
|
||||
supply_chain_report,
|
||||
required=supply_chain_required,
|
||||
)
|
||||
|
||||
return {
|
||||
"tests": tests_status,
|
||||
"coverage": coverage_status,
|
||||
"loc": loc_status,
|
||||
"docs_naming": docs_naming_status,
|
||||
"gate_glue": gate_glue_status,
|
||||
"sonarqube": sonarqube_status,
|
||||
"semgrep": semgrep_status,
|
||||
"supply_chain": supply_chain_status,
|
||||
}
|
||||
@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a compact Semgrep gate report from Semgrep JSON output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
BLOCKING_SEVERITIES = {"ERROR"}
|
||||
SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
|
||||
SONAR_SEVERITY_BY_SEMGREP = {
|
||||
"ERROR": "CRITICAL",
|
||||
"WARNING": "MAJOR",
|
||||
"INFO": "MINOR",
|
||||
"UNKNOWN": "INFO",
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
"""Read a JSON object from disk, returning an error-shaped payload on failure."""
|
||||
if not path.exists():
|
||||
return {"errors": [{"message": f"report missing: {path}"}], "results": []}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
|
||||
if not isinstance(payload, dict):
|
||||
return {"errors": [{"message": "report payload is not an object"}], "results": []}
|
||||
return payload
|
||||
|
||||
|
||||
def _finding_severity(finding: dict[str, Any]) -> str:
|
||||
"""Return a normalized Semgrep finding severity."""
|
||||
extra = finding.get("extra")
|
||||
severity = extra.get("severity") if isinstance(extra, dict) else None
|
||||
normalized = str(severity or "UNKNOWN").strip().upper()
|
||||
return normalized if normalized in SEVERITIES else "UNKNOWN"
|
||||
|
||||
|
||||
def build_report(
|
||||
semgrep_payload: dict[str, Any],
|
||||
*,
|
||||
semgrep_exit_code: int,
|
||||
blocking_severities: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Summarize Semgrep evidence into the quality-gate report contract."""
|
||||
blocking = blocking_severities or BLOCKING_SEVERITIES
|
||||
raw_results = semgrep_payload.get("results", [])
|
||||
raw_errors = semgrep_payload.get("errors", [])
|
||||
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
|
||||
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
|
||||
|
||||
severity_counts = dict.fromkeys(SEVERITIES, 0)
|
||||
blocking_findings = 0
|
||||
for finding in results:
|
||||
severity = _finding_severity(finding)
|
||||
severity_counts[severity] += 1
|
||||
if severity in blocking:
|
||||
blocking_findings += 1
|
||||
|
||||
engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
|
||||
status = "failed" if engine_error or blocking_findings else "ok"
|
||||
return {
|
||||
"status": status,
|
||||
"scanner": "semgrep",
|
||||
"semgrep_rc": semgrep_exit_code,
|
||||
"findings_total": len(results),
|
||||
"blocking_findings": blocking_findings,
|
||||
"errors_total": len(errors),
|
||||
"severity_counts": severity_counts,
|
||||
"blocking_severities": sorted(blocking),
|
||||
}
|
||||
|
||||
|
||||
def _line_number(value: Any, default: int = 1) -> int:
|
||||
"""Return a Sonar-compatible one-indexed line number."""
|
||||
try:
|
||||
line = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return max(line, default)
|
||||
|
||||
|
||||
def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Convert one Semgrep finding into SonarQube generic issue format."""
|
||||
path = str(finding.get("path") or "").strip()
|
||||
if not path:
|
||||
return None
|
||||
extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
|
||||
severity = _finding_severity(finding)
|
||||
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
|
||||
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
|
||||
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
|
||||
# _line_number floors the end line at start_line, so the range never inverts.
|
||||
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
|
||||
return {
|
||||
"engineId": "semgrep",
|
||||
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
|
||||
"type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
|
||||
"severity": SONAR_SEVERITY_BY_SEMGREP[severity],
|
||||
"primaryLocation": {
|
||||
"message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
|
||||
"filePath": path,
|
||||
"textRange": {
|
||||
"startLine": start_line,
|
||||
"endLine": end_line,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Build SonarQube generic external issues from Semgrep JSON output."""
|
||||
raw_results = semgrep_payload.get("results", [])
|
||||
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
|
||||
issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
|
||||
return {"issues": issues}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entrypoint used by Jenkins after Semgrep finishes."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--semgrep-json", required=True)
|
||||
parser.add_argument("--exit-code", type=int, default=0)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--sonar-issues-output")
|
||||
parser.add_argument(
|
||||
"--blocking-severity",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Severity that should mark the report failed. Defaults to ERROR.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
|
||||
payload = _read_json(Path(args.semgrep_json))
|
||||
report = build_report(
|
||||
payload,
|
||||
semgrep_exit_code=args.exit_code,
|
||||
blocking_severities=blocking or BLOCKING_SEVERITIES,
|
||||
)
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
if args.sonar_issues_output:
|
||||
sonar_output = Path(args.sonar_issues_output)
|
||||
sonar_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
sonar_output.write_text(
|
||||
json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@ -1,177 +0,0 @@
|
||||
"""Build a titan-iac supply-chain compliance report from Trivy evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
FAIL_SEVERITIES = {"HIGH", "CRITICAL"}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
"""Read a JSON object from disk for use as pipeline evidence."""
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_day(raw: str | None) -> dt.date | None:
|
||||
"""Parse an ISO day while letting optional waiver dates stay optional."""
|
||||
if not raw:
|
||||
return None
|
||||
return dt.date.fromisoformat(raw)
|
||||
|
||||
|
||||
def _today(override: str | None = None) -> dt.date:
|
||||
"""Return the policy day so tests can pin expiry behavior."""
|
||||
return _parse_day(override) or dt.date.today()
|
||||
|
||||
|
||||
def _load_waiver_pairs(path: Path | None, policy_day: dt.date) -> tuple[set[tuple[str, str]], int]:
|
||||
"""Return active ``(misconfiguration id, target)`` waivers and expired count."""
|
||||
if path is None or not path.exists():
|
||||
return set(), 0
|
||||
|
||||
payload = _read_json(path)
|
||||
default_expires_at = payload.get("default_expires_at")
|
||||
active: set[tuple[str, str]] = set()
|
||||
expired = 0
|
||||
|
||||
for entry in payload.get("misconfigurations", []):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
misconfiguration_id = str(entry.get("id") or "").strip()
|
||||
if not misconfiguration_id:
|
||||
continue
|
||||
expires_at = _parse_day(str(entry.get("expires_at") or default_expires_at or ""))
|
||||
targets = entry.get("targets", [])
|
||||
if not isinstance(targets, list):
|
||||
continue
|
||||
|
||||
if expires_at and expires_at < policy_day:
|
||||
expired += len(targets)
|
||||
continue
|
||||
|
||||
# Waivers are target-specific so a new unsafe manifest fails until it is
|
||||
# either fixed or deliberately accepted with a fresh expiration.
|
||||
for target in targets:
|
||||
if isinstance(target, str) and target:
|
||||
active.add((misconfiguration_id, target))
|
||||
|
||||
return active, expired
|
||||
|
||||
|
||||
def _iter_failed_misconfigurations(payload: dict[str, Any]):
|
||||
"""Yield failed high/critical Trivy misconfiguration records."""
|
||||
for result in payload.get("Results", []):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
target = str(result.get("Target") or "")
|
||||
for item in result.get("Misconfigurations") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("Status") != "FAIL":
|
||||
continue
|
||||
if str(item.get("Severity") or "").upper() not in FAIL_SEVERITIES:
|
||||
continue
|
||||
yield target, item
|
||||
|
||||
|
||||
def _count_vulnerabilities(payload: dict[str, Any], severity: str) -> int:
|
||||
"""Count Trivy vulnerabilities at a specific severity."""
|
||||
count = 0
|
||||
for result in payload.get("Results", []):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
for item in result.get("Vulnerabilities") or []:
|
||||
if isinstance(item, dict) and str(item.get("Severity") or "").upper() == severity:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _count_secrets(payload: dict[str, Any]) -> int:
|
||||
"""Count detected secrets in the Trivy filesystem report."""
|
||||
count = 0
|
||||
for result in payload.get("Results", []):
|
||||
if isinstance(result, dict):
|
||||
count += len(result.get("Secrets") or [])
|
||||
return count
|
||||
|
||||
|
||||
def build_report(
|
||||
trivy_payload: dict[str, Any],
|
||||
waiver_path: Path | None = None,
|
||||
today_override: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the compliance summary consumed by the quality gate."""
|
||||
policy_day = _today(today_override)
|
||||
active_waivers, expired_waivers = _load_waiver_pairs(waiver_path, policy_day)
|
||||
|
||||
open_misconfigs: list[dict[str, str]] = []
|
||||
waived_misconfigs = 0
|
||||
for target, item in _iter_failed_misconfigurations(trivy_payload):
|
||||
misconfiguration_id = str(item.get("ID") or "")
|
||||
if (misconfiguration_id, target) in active_waivers:
|
||||
waived_misconfigs += 1
|
||||
continue
|
||||
open_misconfigs.append(
|
||||
{
|
||||
"id": misconfiguration_id,
|
||||
"target": target,
|
||||
"severity": str(item.get("Severity") or ""),
|
||||
"title": str(item.get("Title") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
|
||||
high = _count_vulnerabilities(trivy_payload, "HIGH")
|
||||
secrets = _count_secrets(trivy_payload)
|
||||
status = (
|
||||
"ok"
|
||||
if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
|
||||
else "failed"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"compliant": status == "ok",
|
||||
"category": "artifact_security",
|
||||
"scan_type": "filesystem",
|
||||
"scanner": "trivy",
|
||||
"critical_vulnerabilities": critical,
|
||||
"high_vulnerabilities": high,
|
||||
"high_vulnerability_policy": "observe",
|
||||
"secrets": secrets,
|
||||
"high_or_critical_misconfigurations": len(open_misconfigs),
|
||||
"waived_misconfigurations": waived_misconfigs,
|
||||
"expired_waivers": expired_waivers,
|
||||
"waiver_file": str(waiver_path) if waiver_path else "",
|
||||
"open_misconfiguration_examples": open_misconfigs[:20],
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entrypoint used by Jenkins after the Trivy scan completes."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--trivy-json", required=True)
|
||||
parser.add_argument("--waivers")
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--today")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
trivy_payload = _read_json(Path(args.trivy_json))
|
||||
waiver_path = Path(args.waivers) if args.waivers else None
|
||||
report = build_report(trivy_payload, waiver_path=waiver_path, today_override=args.today)
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@ -1,18 +0,0 @@
|
||||
max_success_age_hours: 48
|
||||
allow_suspended:
|
||||
- bstein-dev-home/vaultwarden-cred-sync
|
||||
- comms/guest-name-randomizer
|
||||
- comms/othrys-room-reset
|
||||
- comms/pin-othrys-invite
|
||||
- comms/seed-othrys-room
|
||||
- finance/firefly-user-sync
|
||||
- health/wger-admin-ensure
|
||||
- health/wger-user-sync
|
||||
- mailu-mailserver/mailu-sync-nightly
|
||||
- nextcloud/nextcloud-mail-sync
|
||||
- vault/vault-oidc-config
|
||||
ariadne_schedule_tasks:
|
||||
- schedule.mailu_sync
|
||||
- schedule.nextcloud_sync
|
||||
- schedule.vaultwarden_sync
|
||||
- schedule.wger_admin
|
||||
@ -1,34 +0,0 @@
|
||||
"""Skip live glue checks when the VictoriaMetrics endpoint is unavailable."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _endpoint_available() -> bool:
|
||||
"""Return whether the configured VictoriaMetrics host resolves locally."""
|
||||
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
|
||||
parsed = urlparse(vm_url)
|
||||
host = parsed.hostname
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
if not host:
|
||||
return False
|
||||
try:
|
||||
socket.getaddrinfo(host, port)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
"""Mark glue tests skipped when the VictoriaMetrics host cannot resolve."""
|
||||
del config
|
||||
if _endpoint_available():
|
||||
return
|
||||
skip_marker = pytest.mark.skip(reason="VictoriaMetrics endpoint is not reachable in this workspace")
|
||||
for item in items:
|
||||
item.add_marker(skip_marker)
|
||||
@ -1,108 +0,0 @@
|
||||
"""Glue checks for Ariadne schedules exported to VictoriaMetrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
|
||||
CONFIG_PATH = Path(__file__).with_name("config.yaml")
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def _query(promql: str) -> list[dict]:
|
||||
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
|
||||
response = requests.get(f"{vm_url}/api/v1/query", params={"query": promql}, timeout=10)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return payload.get("data", {}).get("result", [])
|
||||
|
||||
|
||||
def _expected_tasks() -> list[dict]:
|
||||
cfg = _load_config()
|
||||
tasks = [
|
||||
_normalize_task(item, cfg)
|
||||
for item in cfg.get("ariadne_schedule_tasks", [])
|
||||
]
|
||||
assert tasks, "No Ariadne schedule tasks configured"
|
||||
return tasks
|
||||
|
||||
|
||||
def _normalize_task(item: object, cfg: dict) -> dict:
|
||||
if isinstance(item, str):
|
||||
return {
|
||||
"task": item,
|
||||
"check_last_success": True,
|
||||
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
|
||||
}
|
||||
if isinstance(item, dict):
|
||||
normalized = dict(item)
|
||||
normalized.setdefault("check_last_success", True)
|
||||
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
|
||||
return normalized
|
||||
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
|
||||
|
||||
|
||||
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
|
||||
tracked = [item for item in tasks if item.get("check_last_success")]
|
||||
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
|
||||
return tracked
|
||||
|
||||
|
||||
def _task_regex(tasks: list[dict]) -> str:
|
||||
return "|".join(item["task"] for item in tasks)
|
||||
|
||||
|
||||
def test_ariadne_schedule_series_exist():
|
||||
tasks = _expected_tasks()
|
||||
selector = _task_regex(tasks)
|
||||
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
|
||||
seen = {item.get("metric", {}).get("task") for item in series}
|
||||
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
||||
assert not missing, f"Missing next-run metrics for: {', '.join(missing)}"
|
||||
|
||||
|
||||
def test_ariadne_schedule_recent_success():
|
||||
tasks = _tracked_tasks(_expected_tasks())
|
||||
selector = _task_regex(tasks)
|
||||
series = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
|
||||
seen = {item.get("metric", {}).get("task") for item in series}
|
||||
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
||||
assert not missing, f"Missing last-success metrics for: {', '.join(missing)}"
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
age_by_task = {
|
||||
item.get("metric", {}).get("task"): (now - datetime.fromtimestamp(float(item["value"][1]), tz=timezone.utc)).total_seconds() / 3600
|
||||
for item in series
|
||||
}
|
||||
too_old = [
|
||||
f"{task} ({age_by_task[task]:.1f}h > {item['max_success_age_hours']}h)"
|
||||
for item in tasks
|
||||
if (task := item["task"]) in age_by_task and age_by_task[task] > float(item["max_success_age_hours"])
|
||||
]
|
||||
assert not too_old, "Ariadne schedules are stale: " + ", ".join(too_old)
|
||||
|
||||
|
||||
def test_ariadne_schedule_last_status_present_and_boolean():
|
||||
tasks = _tracked_tasks(_expected_tasks())
|
||||
selector = _task_regex(tasks)
|
||||
series = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
|
||||
seen = {item.get("metric", {}).get("task") for item in series}
|
||||
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
||||
assert not missing, f"Missing last-status metrics for: {', '.join(missing)}"
|
||||
|
||||
invalid = []
|
||||
for item in series:
|
||||
task = item.get("metric", {}).get("task")
|
||||
value = float(item["value"][1])
|
||||
if value not in (0.0, 1.0):
|
||||
invalid.append(f"{task}={value}")
|
||||
assert not invalid, f"Unexpected Ariadne last-status values: {', '.join(invalid)}"
|
||||
@ -1,46 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from kubernetes import client, config
|
||||
|
||||
|
||||
CONFIG_PATH = Path(__file__).with_name("config.yaml")
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def _load_kube():
|
||||
try:
|
||||
config.load_incluster_config()
|
||||
except config.ConfigException:
|
||||
config.load_kube_config()
|
||||
|
||||
|
||||
def test_glue_cronjobs_recent_success():
|
||||
cfg = _load_config()
|
||||
max_age_hours = int(cfg.get("max_success_age_hours", 48))
|
||||
allow_suspended = set(cfg.get("allow_suspended", []))
|
||||
|
||||
_load_kube()
|
||||
batch = client.BatchV1Api()
|
||||
cronjobs = batch.list_cron_job_for_all_namespaces(label_selector="atlas.bstein.dev/glue=true").items
|
||||
|
||||
assert cronjobs, "No glue cronjobs found with atlas.bstein.dev/glue=true"
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
for cronjob in cronjobs:
|
||||
name = f"{cronjob.metadata.namespace}/{cronjob.metadata.name}"
|
||||
if cronjob.spec.suspend:
|
||||
assert name in allow_suspended, f"{name} is suspended but not in allow_suspended"
|
||||
continue
|
||||
|
||||
last_success = cronjob.status.last_successful_time
|
||||
assert last_success is not None, f"{name} has no lastSuccessfulTime"
|
||||
age_hours = (now - last_success).total_seconds() / 3600
|
||||
assert age_hours <= max_age_hours, f"{name} last success {age_hours:.1f}h ago"
|
||||
@ -1,87 +0,0 @@
|
||||
"""Glue checks for the metrics the quality-gate publishes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
|
||||
VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
|
||||
CONFIG_PATH = Path(__file__).with_name("config.yaml")
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def _query(promql: str) -> list[dict]:
|
||||
response = requests.get(f"{VM_URL}/api/v1/query", params={"query": promql}, timeout=10)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return payload.get("data", {}).get("result", [])
|
||||
|
||||
|
||||
def _expected_tasks() -> list[dict]:
|
||||
cfg = _load_config()
|
||||
tasks = [
|
||||
_normalize_task(item, cfg)
|
||||
for item in cfg.get("ariadne_schedule_tasks", [])
|
||||
]
|
||||
assert tasks, "No Ariadne schedule tasks configured"
|
||||
return tasks
|
||||
|
||||
|
||||
def _normalize_task(item: object, cfg: dict) -> dict:
|
||||
if isinstance(item, str):
|
||||
return {
|
||||
"task": item,
|
||||
"check_last_success": True,
|
||||
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
|
||||
}
|
||||
if isinstance(item, dict):
|
||||
normalized = dict(item)
|
||||
normalized.setdefault("check_last_success", True)
|
||||
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
|
||||
return normalized
|
||||
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
|
||||
|
||||
|
||||
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
|
||||
tracked = [item for item in tasks if item.get("check_last_success")]
|
||||
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
|
||||
return tracked
|
||||
|
||||
|
||||
def _task_regex(tasks: list[dict]) -> str:
|
||||
return "|".join(item["task"] for item in tasks)
|
||||
|
||||
|
||||
def test_ariadne_schedule_metrics_present():
|
||||
tasks = _expected_tasks()
|
||||
selector = _task_regex(tasks)
|
||||
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
|
||||
seen = {item.get("metric", {}).get("task") for item in series}
|
||||
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
||||
assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}"
|
||||
|
||||
|
||||
def test_ariadne_schedule_success_and_status_metrics_present():
|
||||
tasks = _tracked_tasks(_expected_tasks())
|
||||
selector = _task_regex(tasks)
|
||||
|
||||
success = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
|
||||
status = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
|
||||
|
||||
success_tasks = {item.get("metric", {}).get("task") for item in success}
|
||||
status_tasks = {item.get("metric", {}).get("task") for item in status}
|
||||
expected = {item["task"] for item in tasks}
|
||||
|
||||
missing_success = sorted(expected - success_tasks)
|
||||
missing_status = sorted(expected - status_tasks)
|
||||
|
||||
assert not missing_success, f"Missing Ariadne success metrics for: {', '.join(missing_success)}"
|
||||
assert not missing_status, f"Missing Ariadne status metrics for: {', '.join(missing_status)}"
|
||||
@ -1,405 +0,0 @@
|
||||
{
|
||||
"version": 1,
|
||||
"generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
|
||||
"default_expires_at": "2026-09-30",
|
||||
"ticket": "atlas-quality-wave-k8s-hardening",
|
||||
"default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
|
||||
"misconfigurations": [
|
||||
{
|
||||
"id": "DS-0002",
|
||||
"targets": [
|
||||
"dockerfiles/Dockerfile.ananke-node-helper"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0009",
|
||||
"targets": [
|
||||
"services/mailu/vip-controller.yaml",
|
||||
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0010",
|
||||
"targets": [
|
||||
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
|
||||
"services/monitoring/jetson-tegrastats-exporter.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0014",
|
||||
"targets": [
|
||||
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
|
||||
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
|
||||
"infrastructure/core/ntp-sync-daemonset.yaml",
|
||||
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
|
||||
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
|
||||
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
|
||||
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
|
||||
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
|
||||
"infrastructure/postgres/statefulset.yaml",
|
||||
"infrastructure/vault-csi/vault-csi-provider.yaml",
|
||||
"services/ai-llm/deployment.yaml",
|
||||
"services/bstein-dev-home/backend-deployment.yaml",
|
||||
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
|
||||
"services/bstein-dev-home/frontend-deployment.yaml",
|
||||
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
|
||||
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
|
||||
"services/bstein-dev-home/vault-sync-deployment.yaml",
|
||||
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
|
||||
"services/comms/atlasbot-deployment.yaml",
|
||||
"services/comms/coturn.yaml",
|
||||
"services/comms/element-call-deployment.yaml",
|
||||
"services/comms/guest-name-job.yaml",
|
||||
"services/comms/guest-register-deployment.yaml",
|
||||
"services/comms/livekit-token-deployment.yaml",
|
||||
"services/comms/livekit.yaml",
|
||||
"services/comms/mas-deployment.yaml",
|
||||
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
|
||||
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
|
||||
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
|
||||
"services/comms/pin-othrys-job.yaml",
|
||||
"services/comms/reset-othrys-room-job.yaml",
|
||||
"services/comms/seed-othrys-room.yaml",
|
||||
"services/comms/vault-sync-deployment.yaml",
|
||||
"services/comms/wellknown.yaml",
|
||||
"services/crypto/monerod/deployment.yaml",
|
||||
"services/crypto/wallet-monero-temp/deployment.yaml",
|
||||
"services/crypto/xmr-miner/deployment.yaml",
|
||||
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
|
||||
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
|
||||
"services/finance/actual-budget-deployment.yaml",
|
||||
"services/finance/firefly-cronjob.yaml",
|
||||
"services/finance/firefly-deployment.yaml",
|
||||
"services/finance/firefly-user-sync-cronjob.yaml",
|
||||
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
|
||||
"services/gitea/deployment.yaml",
|
||||
"services/harbor/vault-sync-deployment.yaml",
|
||||
"services/health/wger-admin-ensure-cronjob.yaml",
|
||||
"services/health/wger-deployment.yaml",
|
||||
"services/health/wger-user-sync-cronjob.yaml",
|
||||
"services/jellyfin/deployment.yaml",
|
||||
"services/jellyfin/loader.yaml",
|
||||
"services/jenkins/deployment.yaml",
|
||||
"services/jenkins/vault-sync-deployment.yaml",
|
||||
"services/keycloak/deployment.yaml",
|
||||
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/vault-sync-deployment.yaml",
|
||||
"services/logging/node-image-gc-rpi4-daemonset.yaml",
|
||||
"services/logging/node-image-prune-rpi5-daemonset.yaml",
|
||||
"services/logging/node-log-rotation-daemonset.yaml",
|
||||
"services/logging/oauth2-proxy.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
|
||||
"services/logging/opensearch-prune-cronjob.yaml",
|
||||
"services/logging/vault-sync-deployment.yaml",
|
||||
"services/mailu/mailu-sync-cronjob.yaml",
|
||||
"services/mailu/mailu-sync-listener.yaml",
|
||||
"services/mailu/vault-sync-deployment.yaml",
|
||||
"services/mailu/vip-controller.yaml",
|
||||
"services/maintenance/apps/ariadne-deployment.yaml",
|
||||
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
|
||||
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
|
||||
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
|
||||
"services/maintenance/apps/metis-deployment.yaml",
|
||||
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
|
||||
"services/maintenance/networking/oauth2-proxy-metis.yaml",
|
||||
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
|
||||
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
|
||||
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
|
||||
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
|
||||
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
|
||||
"services/maintenance/apps/soteria-deployment.yaml",
|
||||
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
|
||||
"services/monitoring/dcgm-exporter.yaml",
|
||||
"services/monitoring/jetson-tegrastats-exporter.yaml",
|
||||
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
|
||||
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
|
||||
"services/monitoring/platform-quality-gateway-deployment.yaml",
|
||||
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
|
||||
"services/monitoring/postmark-exporter-deployment.yaml",
|
||||
"services/monitoring/vmalert-atlas-availability.yaml",
|
||||
"services/monitoring/vault-sync-deployment.yaml",
|
||||
"services/nextcloud-mail-sync/cronjob.yaml",
|
||||
"services/nextcloud/collabora.yaml",
|
||||
"services/nextcloud/cronjob.yaml",
|
||||
"services/nextcloud/deployment.yaml",
|
||||
"services/nextcloud/maintenance-cronjob.yaml",
|
||||
"services/oauth2-proxy/deployment.yaml",
|
||||
"services/openldap/statefulset.yaml",
|
||||
"services/outline/deployment.yaml",
|
||||
"services/outline/redis-deployment.yaml",
|
||||
"services/pegasus/deployment.yaml",
|
||||
"services/pegasus/vault-sync-deployment.yaml",
|
||||
"services/planka/deployment.yaml",
|
||||
"services/quality/oauth2-proxy-sonarqube.yaml",
|
||||
"services/quality/sonarqube-deployment.yaml",
|
||||
"services/quality/sonarqube-exporter-deployment.yaml",
|
||||
"services/sui-metrics/base/deployment.yaml",
|
||||
"services/typhon/vault-sync-deployment.yaml",
|
||||
"services/vault/k8s-auth-config-cronjob.yaml",
|
||||
"services/vault/oidc-config-cronjob.yaml",
|
||||
"services/vault/statefulset.yaml",
|
||||
"services/vaultwarden/deployment.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0017",
|
||||
"targets": [
|
||||
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
|
||||
"services/logging/node-image-gc-rpi4-daemonset.yaml",
|
||||
"services/logging/node-image-prune-rpi5-daemonset.yaml",
|
||||
"services/logging/node-log-rotation-daemonset.yaml",
|
||||
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
|
||||
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
|
||||
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
|
||||
"services/maintenance/apps/metis-deployment.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
|
||||
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
|
||||
"services/monitoring/dcgm-exporter.yaml",
|
||||
"services/monitoring/jetson-tegrastats-exporter.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0041",
|
||||
"targets": [
|
||||
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
|
||||
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
|
||||
"infrastructure/traefik/clusterrole.yaml",
|
||||
"services/bstein-dev-home/rbac.yaml",
|
||||
"services/comms/comms-secrets-ensure-rbac.yaml",
|
||||
"services/comms/mas-db-ensure-rbac.yaml",
|
||||
"services/comms/mas-secrets-ensure-rbac.yaml",
|
||||
"services/maintenance/apps/soteria-rbac.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0047",
|
||||
"targets": [
|
||||
"services/monitoring/rbac.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0053",
|
||||
"targets": [
|
||||
"services/comms/comms-secrets-ensure-rbac.yaml",
|
||||
"services/comms/mas-db-ensure-rbac.yaml",
|
||||
"services/jenkins/serviceaccount.yaml",
|
||||
"services/maintenance/apps/ariadne-rbac.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0056",
|
||||
"targets": [
|
||||
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
|
||||
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
|
||||
"services/jenkins/serviceaccount.yaml",
|
||||
"services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml",
|
||||
"services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0114",
|
||||
"targets": [
|
||||
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0118",
|
||||
"targets": [
|
||||
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
|
||||
"infrastructure/core/coredns-deployment.yaml",
|
||||
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
|
||||
"infrastructure/core/ntp-sync-daemonset.yaml",
|
||||
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
|
||||
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
|
||||
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
|
||||
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
|
||||
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
|
||||
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
|
||||
"infrastructure/postgres/statefulset.yaml",
|
||||
"infrastructure/vault-csi/vault-csi-provider.yaml",
|
||||
"services/ai-llm/deployment.yaml",
|
||||
"services/bstein-dev-home/backend-deployment.yaml",
|
||||
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
|
||||
"services/bstein-dev-home/frontend-deployment.yaml",
|
||||
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
|
||||
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
|
||||
"services/bstein-dev-home/vault-sync-deployment.yaml",
|
||||
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
|
||||
"services/comms/atlasbot-deployment.yaml",
|
||||
"services/comms/coturn.yaml",
|
||||
"services/comms/element-call-deployment.yaml",
|
||||
"services/comms/guest-name-job.yaml",
|
||||
"services/comms/livekit-token-deployment.yaml",
|
||||
"services/comms/livekit.yaml",
|
||||
"services/comms/mas-deployment.yaml",
|
||||
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
|
||||
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
|
||||
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
|
||||
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
|
||||
"services/comms/pin-othrys-job.yaml",
|
||||
"services/comms/reset-othrys-room-job.yaml",
|
||||
"services/comms/seed-othrys-room.yaml",
|
||||
"services/comms/vault-sync-deployment.yaml",
|
||||
"services/comms/wellknown.yaml",
|
||||
"services/crypto/monerod/deployment.yaml",
|
||||
"services/crypto/wallet-monero-temp/deployment.yaml",
|
||||
"services/crypto/xmr-miner/deployment.yaml",
|
||||
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
|
||||
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
|
||||
"services/finance/firefly-cronjob.yaml",
|
||||
"services/finance/firefly-deployment.yaml",
|
||||
"services/finance/firefly-user-sync-cronjob.yaml",
|
||||
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
|
||||
"services/gitea/deployment.yaml",
|
||||
"services/harbor/vault-sync-deployment.yaml",
|
||||
"services/health/wger-admin-ensure-cronjob.yaml",
|
||||
"services/health/wger-deployment.yaml",
|
||||
"services/health/wger-user-sync-cronjob.yaml",
|
||||
"services/jellyfin/loader.yaml",
|
||||
"services/jenkins/deployment.yaml",
|
||||
"services/jenkins/vault-sync-deployment.yaml",
|
||||
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
|
||||
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
|
||||
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
|
||||
"services/keycloak/vault-sync-deployment.yaml",
|
||||
"services/logging/node-image-gc-rpi4-daemonset.yaml",
|
||||
"services/logging/node-image-prune-rpi5-daemonset.yaml",
|
||||
"services/logging/node-log-rotation-daemonset.yaml",
|
||||
"services/logging/oauth2-proxy.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
|
||||
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
|
||||
"services/logging/opensearch-prune-cronjob.yaml",
|
||||
"services/logging/vault-sync-deployment.yaml",
|
||||
"services/mailu/mailu-sync-cronjob.yaml",
|
||||
"services/mailu/mailu-sync-listener.yaml",
|
||||
"services/mailu/vault-sync-deployment.yaml",
|
||||
"services/mailu/vip-controller.yaml",
|
||||
"services/maintenance/apps/ariadne-deployment.yaml",
|
||||
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
|
||||
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
|
||||
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
|
||||
"services/maintenance/apps/metis-deployment.yaml",
|
||||
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
|
||||
"services/maintenance/networking/oauth2-proxy-metis.yaml",
|
||||
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
|
||||
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
|
||||
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
|
||||
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
|
||||
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
|
||||
"services/maintenance/apps/soteria-deployment.yaml",
|
||||
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
|
||||
"services/monitoring/dcgm-exporter.yaml",
|
||||
"services/monitoring/jetson-tegrastats-exporter.yaml",
|
||||
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
|
||||
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
|
||||
"services/monitoring/platform-quality-gateway-deployment.yaml",
|
||||
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
|
||||
"services/monitoring/postmark-exporter-deployment.yaml",
|
||||
"services/monitoring/vmalert-atlas-availability.yaml",
|
||||
"services/monitoring/vault-sync-deployment.yaml",
|
||||
"services/nextcloud/collabora.yaml",
|
||||
"services/oauth2-proxy/deployment.yaml",
|
||||
"services/openldap/statefulset.yaml",
|
||||
"services/outline/deployment.yaml",
|
||||
"services/outline/redis-deployment.yaml",
|
||||
"services/pegasus/vault-sync-deployment.yaml",
|
||||
"services/quality/oauth2-proxy-sonarqube.yaml",
|
||||
"services/quality/sonarqube-deployment.yaml",
|
||||
"services/quality/sonarqube-exporter-deployment.yaml",
|
||||
"services/sui-metrics/base/deployment.yaml",
|
||||
"services/sui-metrics/overlays/atlas/patch-node-selector.yaml",
|
||||
"services/typhon/deployment.yaml",
|
||||
"services/typhon/vault-sync-deployment.yaml",
|
||||
"services/vault/k8s-auth-config-cronjob.yaml",
|
||||
"services/vault/oidc-config-cronjob.yaml",
|
||||
"services/vaultwarden/deployment.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "KSV-0121",
|
||||
"targets": [
|
||||
"services/logging/node-image-gc-rpi4-daemonset.yaml",
|
||||
"services/logging/node-image-prune-rpi5-daemonset.yaml",
|
||||
"services/logging/node-log-rotation-daemonset.yaml",
|
||||
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
|
||||
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
|
||||
"services/maintenance/apps/metis-deployment.yaml",
|
||||
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
|
||||
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
|
||||
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,6 +0,0 @@
|
||||
# clusters/aether/flux-system/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- platform
|
||||
- applications
|
||||
12
clusters/atlas/applications/kustomization.yaml
Normal file
12
clusters/atlas/applications/kustomization.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
# clusters/atlas/applications/kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../services/crypto
|
||||
- ../../services/gitea
|
||||
- ../../services/jellyfin
|
||||
- ../../services/jitsi
|
||||
- ../../services/monitoring
|
||||
- ../../services/pegasus
|
||||
- ../../services/vault
|
||||
- ../../services/bstein-dev-home
|
||||
@ -1,27 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/ai-llm/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: ai-llm
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: Merge
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: false
|
||||
timeout: 30m
|
||||
path: ./services/ai-llm
|
||||
targetNamespace: ai
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: ollama
|
||||
namespace: ai
|
||||
dependsOn:
|
||||
- name: core
|
||||
@ -1,20 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: bstein-dev-home-migrations
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/bstein-dev-home/migration-jobs
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: bstein-dev-home
|
||||
wait: false
|
||||
suspend: true
|
||||
@ -3,7 +3,7 @@ apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: bstein-dev-home
|
||||
namespace: bstein-dev-home
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
@ -13,14 +13,14 @@ spec:
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
branch: feature/ci-gitops
|
||||
commit:
|
||||
author:
|
||||
email: ops@bstein.dev
|
||||
name: flux-bot
|
||||
messageTemplate: "chore(bstein-dev-home): automated image update"
|
||||
messageTemplate: "chore(bstein-dev-home): update images to {{range .Updated.Images}}{{.}}{{end}}"
|
||||
push:
|
||||
branch: main
|
||||
branch: feature/ci-gitops
|
||||
update:
|
||||
strategy: Setters
|
||||
path: services/bstein-dev-home
|
||||
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: bstein-dev-home
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/bstein-dev-home
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/cassandra-auth/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cassandra-auth
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/cassandra-auth
|
||||
targetNamespace: sso
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: keycloak
|
||||
- name: vault
|
||||
- name: vault-injector
|
||||
wait: false
|
||||
timeout: 10m
|
||||
@ -1,29 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/cassandra/image-automation.yaml
|
||||
# Staged for the first Cassandra image rollout. Add this file to the parent
|
||||
# applications kustomization after the namespace exists and the Harbor repos
|
||||
# have initial tags.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: cassandra
|
||||
namespace: cassandra
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
commit:
|
||||
author:
|
||||
email: ops@bstein.dev
|
||||
name: flux-bot
|
||||
messageTemplate: "chore(cassandra): automated image update"
|
||||
push:
|
||||
branch: main
|
||||
update:
|
||||
strategy: Setters
|
||||
path: services/cassandra
|
||||
@ -1,28 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/cassandra/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cassandra
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/cassandra
|
||||
targetNamespace: cassandra
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: core
|
||||
- name: keycloak
|
||||
- name: longhorn
|
||||
- name: traefik
|
||||
- name: vault
|
||||
- name: vault-csi
|
||||
- name: vault-injector
|
||||
wait: false
|
||||
timeout: 20m
|
||||
@ -1,9 +1,9 @@
|
||||
# clusters/atlas/flux-system/platform/maintenance/image-automation.yaml
|
||||
# clusters/atlas/flux-system/applications/ci-demo/image-automation.yaml
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: maintenance
|
||||
namespace: maintenance
|
||||
name: ci-demo
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
@ -13,14 +13,14 @@ spec:
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
branch: feature/ci-gitops
|
||||
commit:
|
||||
author:
|
||||
email: ops@bstein.dev
|
||||
name: flux-bot
|
||||
messageTemplate: "chore(maintenance): automated image update"
|
||||
messageTemplate: "chore(ci-demo): apply image updates"
|
||||
push:
|
||||
branch: main
|
||||
branch: feature/ci-gitops
|
||||
update:
|
||||
strategy: Setters
|
||||
path: services/maintenance
|
||||
path: services/ci-demo
|
||||
@ -1,16 +1,17 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
|
||||
# clusters/atlas/flux-system/applications/ci-demo/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-triage-demo
|
||||
name: ci-demo
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-triage-demo
|
||||
path: ./services/ci-demo
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
wait: false
|
||||
timeout: 5m
|
||||
@ -1,17 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/comms/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: comms
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./services/comms
|
||||
targetNamespace: comms
|
||||
timeout: 2m
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: crypto
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/crypto
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/finance/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: finance
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/finance
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: finance
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: actual-budget
|
||||
namespace: finance
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: firefly
|
||||
namespace: finance
|
||||
wait: false
|
||||
@ -1,31 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/game-stream/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: game-stream
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/game-stream
|
||||
targetNamespace: game-stream
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: keycloak
|
||||
- name: traefik
|
||||
- name: vault
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: oauth2-proxy-wolf
|
||||
namespace: game-stream
|
||||
wait: false
|
||||
timeout: 10m
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/gitea
|
||||
@ -15,8 +13,4 @@ spec:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: postgres
|
||||
wait: true
|
||||
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: harbor
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/harbor
|
||||
@ -15,23 +13,11 @@ spec:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: harbor-hermes-agent-immutability-ensure-1
|
||||
namespace: harbor
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: harbor-hermes-webui-immutability-ensure-1
|
||||
namespace: harbor
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: harbor-hermes-chat-router-immutability-ensure-1
|
||||
- apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
name: harbor
|
||||
namespace: harbor
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: postgres
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/health/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: health
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/health
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: health
|
||||
dependsOn:
|
||||
- name: keycloak
|
||||
- name: postgres
|
||||
- name: vault
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: wger
|
||||
namespace: health
|
||||
wait: false
|
||||
@ -1,21 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-chat
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-chat
|
||||
targetNamespace: hermes-chat
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 30m
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: hermes
|
||||
- name: longhorn
|
||||
@ -1,19 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-observer-bindings
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-observer-bindings
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: hermes
|
||||
- name: hermes-observer-rbac
|
||||
@ -1,16 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-observer-rbac
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-observer-rbac
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
@ -1,26 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-scm-broker
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-scm-broker
|
||||
targetNamespace: hermes-scm
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-scm-broker
|
||||
namespace: hermes-scm
|
||||
dependsOn:
|
||||
- name: vault
|
||||
- name: gitea
|
||||
- name: hermes-scm-namespace
|
||||
@ -1,26 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes/image-automation.yaml
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: hermes
|
||||
namespace: hermes
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
commit:
|
||||
author:
|
||||
email: ops@bstein.dev
|
||||
name: flux-bot
|
||||
messageTemplate: "chore(hermes): promote validated image release"
|
||||
push:
|
||||
branch: main
|
||||
update:
|
||||
strategy: Setters
|
||||
path: services/hermes
|
||||
@ -1,69 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/hermes/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: Merge
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes
|
||||
targetNamespace: hermes
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: false
|
||||
timeout: 10m
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-switchyard
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-agent
|
||||
namespace: hermes
|
||||
# hermes-execution-worker and its mediators are deliberately absent. They run
|
||||
# at scavenger priority on a best-effort pool whose first start installs the
|
||||
# provider CLIs, so gating this Kustomization's 10m health window on them
|
||||
# would stall hermes-chat and hermes-observer-bindings, which dependsOn
|
||||
# hermes. The pool reports its own health through the worker readiness probe,
|
||||
# the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
|
||||
# The node SSH hardener is deliberately absent. It reconciles every node,
|
||||
# including offline and maintenance hosts, so its availability must not
|
||||
# turn a node-local account issue into a blocked owner-service rollout.
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
name: hermes-chat-tenant
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-chat-router
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: oauth2-proxy-hermes-chat
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: oauth2-proxy-hermes-triage
|
||||
namespace: hermes
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: core
|
||||
- name: keycloak
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: jenkins
|
||||
- name: hermes-observer-rbac
|
||||
@ -4,12 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: jellyfin
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/jellyfin
|
||||
targetNamespace: jellyfin
|
||||
prune: true
|
||||
@ -19,6 +15,5 @@ spec:
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: openldap
|
||||
wait: true
|
||||
timeout: 5m
|
||||
|
||||
@ -4,11 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: jenkins
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: Merge
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: false
|
||||
path: ./services/jenkins
|
||||
prune: true
|
||||
sourceRef:
|
||||
@ -17,16 +14,10 @@ spec:
|
||||
targetNamespace: jenkins
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: harbor
|
||||
- name: vault-hermes-jenkins-token-seed
|
||||
- name: traefik
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: jenkins
|
||||
namespace: jenkins
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
- apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
name: jenkins
|
||||
namespace: jenkins
|
||||
wait: false
|
||||
timeout: 5m
|
||||
|
||||
@ -1,16 +1,19 @@
|
||||
# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml
|
||||
# clusters/atlas/flux-system/applications/jitsi/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: hermes-scm-namespace
|
||||
name: jitsi
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/hermes-scm-namespace
|
||||
path: ./services/jitsi
|
||||
targetNamespace: jitsi
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
wait: true
|
||||
timeout: 5m
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
prune: true
|
||||
@ -14,8 +12,4 @@ spec:
|
||||
name: flux-system
|
||||
path: ./services/keycloak
|
||||
targetNamespace: sso
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: postgres
|
||||
timeout: 2m
|
||||
|
||||
@ -4,46 +4,21 @@ kind: Kustomization
|
||||
resources:
|
||||
- gitea/kustomization.yaml
|
||||
- vault/kustomization.yaml
|
||||
- vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
- vaultwarden/kustomization.yaml
|
||||
- comms/kustomization.yaml
|
||||
- jitsi/kustomization.yaml
|
||||
- crypto/kustomization.yaml
|
||||
- monerod/kustomization.yaml
|
||||
- pegasus/kustomization.yaml
|
||||
- pegasus/image-automation.yaml
|
||||
- bstein-dev-home/kustomization.yaml
|
||||
- bstein-dev-home/image-automation.yaml
|
||||
- bstein-dev-home-migrations/kustomization.yaml
|
||||
- harbor/kustomization.yaml
|
||||
- harbor/image-automation.yaml
|
||||
- jellyfin/kustomization.yaml
|
||||
- xmr-miner/kustomization.yaml
|
||||
- wallet-monero-temp/kustomization.yaml
|
||||
- sui-metrics/kustomization.yaml
|
||||
- openldap/kustomization.yaml
|
||||
- keycloak/kustomization.yaml
|
||||
- quality/kustomization.yaml
|
||||
- oauth2-proxy/kustomization.yaml
|
||||
- mailu/kustomization.yaml
|
||||
- jenkins/kustomization.yaml
|
||||
- ai-llm/kustomization.yaml
|
||||
- openclaw/kustomization.yaml
|
||||
- hermes-scm-namespace/kustomization.yaml
|
||||
- hermes/kustomization.yaml
|
||||
- hermes/image-automation.yaml
|
||||
- hermes-observer-rbac/kustomization.yaml
|
||||
- hermes-observer-bindings/kustomization.yaml
|
||||
- hermes-scm-broker/kustomization.yaml
|
||||
- hermes-chat/kustomization.yaml
|
||||
- hermes-triage-demo/kustomization.yaml
|
||||
- game-stream/kustomization.yaml
|
||||
- cassandra-auth/kustomization.yaml
|
||||
- cassandra/kustomization.yaml
|
||||
- cassandra/image-automation.yaml
|
||||
- typhon/kustomization.yaml
|
||||
- nextcloud/kustomization.yaml
|
||||
- nextcloud-mail-sync/kustomization.yaml
|
||||
- outline/kustomization.yaml
|
||||
- planka/kustomization.yaml
|
||||
- finance/kustomization.yaml
|
||||
- health/kustomization.yaml
|
||||
- ci-demo/kustomization.yaml
|
||||
- ci-demo/image-automation.yaml
|
||||
|
||||
@ -4,12 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: mailu
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: monerod
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/crypto/monerod
|
||||
@ -18,4 +16,4 @@ spec:
|
||||
dependsOn:
|
||||
- name: crypto
|
||||
wait: true
|
||||
timeout: 15m
|
||||
timeout: 5m
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: nextcloud-mail-sync
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./services/nextcloud-mail-sync
|
||||
targetNamespace: nextcloud
|
||||
timeout: 2m
|
||||
dependsOn:
|
||||
- name: keycloak
|
||||
@ -1,20 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: nextcloud
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/nextcloud
|
||||
targetNamespace: nextcloud
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: oauth2-proxy
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
prune: true
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/openclaw/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: openclaw
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: Merge
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/openclaw
|
||||
targetNamespace: openclaw
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: longhorn
|
||||
@ -1,21 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/openldap/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: openldap
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./services/openldap
|
||||
targetNamespace: sso
|
||||
dependsOn:
|
||||
- name: core
|
||||
wait: true
|
||||
timeout: 5m
|
||||
@ -1,31 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/outline/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: outline
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/outline
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: outline
|
||||
dependsOn:
|
||||
- name: keycloak
|
||||
- name: mailu
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: outline
|
||||
namespace: outline
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
name: outline
|
||||
namespace: outline
|
||||
wait: false
|
||||
@ -3,7 +3,7 @@ apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: pegasus
|
||||
namespace: jellyfin
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: pegasus
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/pegasus
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/planka/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: planka
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/planka
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: planka
|
||||
dependsOn:
|
||||
- name: keycloak
|
||||
- name: mailu
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: planka
|
||||
namespace: planka
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
name: planka
|
||||
namespace: planka
|
||||
wait: false
|
||||
@ -1,38 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/quality/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: quality
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/quality
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: quality
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: keycloak
|
||||
- name: vault
|
||||
- name: postgres
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: sonarqube
|
||||
namespace: quality
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: sonarqube-exporter
|
||||
namespace: quality
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: oauth2-proxy-sonarqube
|
||||
namespace: quality
|
||||
wait: false
|
||||
timeout: 20m
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: sui-metrics
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/sui-metrics/overlays/atlas
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/typhon/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: typhon
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/typhon
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: climate
|
||||
dependsOn:
|
||||
- name: vault
|
||||
- name: vault-csi
|
||||
- name: monitoring
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: typhon
|
||||
namespace: climate
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
name: typhon
|
||||
namespace: climate
|
||||
wait: false
|
||||
timeout: 20m
|
||||
@ -1,26 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vault-hermes-jenkins-token-seed
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./services/vault-hermes-jenkins-token-seed
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: vault
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-hermes-jenkins-build-token-seed-2
|
||||
namespace: vault
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: vault
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
sourceRef:
|
||||
@ -16,11 +14,5 @@ spec:
|
||||
targetNamespace: vault
|
||||
prune: true
|
||||
wait: true
|
||||
healthChecks:
|
||||
- apiVersion: batch/v1
|
||||
kind: Job
|
||||
name: vault-k8s-auth-hermes-11
|
||||
namespace: vault
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: helm
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vaultwarden
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./services/vaultwarden
|
||||
targetNamespace: vaultwarden
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: helm
|
||||
@ -1,29 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/veles/image-automation.yaml
|
||||
# Staged for the first Veles image rollout. Add this file to the parent
|
||||
# applications kustomization after the namespace exists and the Harbor repos
|
||||
# have initial tags.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: veles
|
||||
namespace: veles
|
||||
spec:
|
||||
interval: 1m0s
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
commit:
|
||||
author:
|
||||
email: ops@bstein.dev
|
||||
name: flux-bot
|
||||
messageTemplate: "chore(veles): automated image update"
|
||||
push:
|
||||
branch: main
|
||||
update:
|
||||
strategy: Setters
|
||||
path: services/veles
|
||||
@ -1,28 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/veles/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: veles
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/veles
|
||||
targetNamespace: veles
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: cert-manager
|
||||
- name: core
|
||||
- name: keycloak
|
||||
- name: longhorn
|
||||
- name: traefik
|
||||
- name: vault
|
||||
- name: vault-csi
|
||||
- name: vault-injector
|
||||
wait: false
|
||||
timeout: 20m
|
||||
@ -1,23 +0,0 @@
|
||||
# clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: wallet-monero-temp
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/crypto/wallet-monero-temp
|
||||
targetNamespace: crypto
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: crypto
|
||||
- name: xmr-miner
|
||||
wait: true
|
||||
@ -4,12 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: xmr-miner
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./services/crypto/xmr-miner
|
||||
targetNamespace: crypto
|
||||
prune: true
|
||||
@ -21,4 +17,3 @@ spec:
|
||||
- name: crypto
|
||||
- name: monerod
|
||||
wait: true
|
||||
timeout: 30m
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
# clusters/atlas/flux-system/gotk-components.yaml
|
||||
---
|
||||
# This manifest was generated by flux. DO NOT EDIT.
|
||||
# Flux Version: v2.7.5
|
||||
@ -5966,9 +5965,6 @@ spec:
|
||||
- args:
|
||||
- --events-addr=http://notification-controller.$(RUNTIME_NAMESPACE).svc.cluster.local./
|
||||
- --watch-all-namespaces=true
|
||||
- --concurrent=4
|
||||
- --requeue-dependency=5s
|
||||
- --interval-jitter-percentage=30
|
||||
- --log-level=info
|
||||
- --log-encoding=json
|
||||
- --enable-leader-election
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
# clusters/atlas/flux-system/gotk-sync.yaml
|
||||
# This manifest was generated by flux. DO NOT EDIT.
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
@ -7,12 +6,12 @@ metadata:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 15m0s
|
||||
interval: 1m0s
|
||||
ref:
|
||||
branch: main
|
||||
branch: feature/ci-gitops
|
||||
secretRef:
|
||||
name: flux-system-gitea
|
||||
url: ssh://git@scm.bstein.dev:2242/titan/atlas-iac.git
|
||||
url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
|
||||
---
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
@ -20,7 +19,7 @@ metadata:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1h0m0s
|
||||
interval: 10m0s
|
||||
path: ./clusters/atlas/flux-system
|
||||
prune: true
|
||||
sourceRef:
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cert-manager-cleanup
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
path: ./infrastructure/cert-manager/cleanup
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
targetNamespace: cert-manager
|
||||
wait: true
|
||||
@ -1,21 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cert-manager
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
path: ./infrastructure/cert-manager
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
targetNamespace: cert-manager
|
||||
dependsOn:
|
||||
- name: helm
|
||||
wait: true
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: core
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./infrastructure/core
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/descheduler/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: descheduler
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
|
||||
spec:
|
||||
interval: 30m
|
||||
suspend: true
|
||||
path: ./infrastructure/descheduler
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
targetNamespace: kube-system
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: core
|
||||
wait: true
|
||||
@ -4,12 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: gitops-ui
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
timeout: 10m
|
||||
path: ./services/gitops-ui
|
||||
prune: true
|
||||
@ -20,4 +16,5 @@ spec:
|
||||
targetNamespace: flux-system
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: traefik
|
||||
wait: true
|
||||
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: helm
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
sourceRef:
|
||||
|
||||
@ -4,19 +4,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- core/kustomization.yaml
|
||||
- helm/kustomization.yaml
|
||||
- descheduler/kustomization.yaml
|
||||
- resource-guardrails/kustomization.yaml
|
||||
- cert-manager/kustomization.yaml
|
||||
- metallb/kustomization.yaml
|
||||
- traefik/kustomization.yaml
|
||||
- gitops-ui/kustomization.yaml
|
||||
- monitoring/kustomization.yaml
|
||||
- logging/kustomization.yaml
|
||||
- maintenance/kustomization.yaml
|
||||
- maintenance/image-automation.yaml
|
||||
- longhorn-adopt/kustomization.yaml
|
||||
- longhorn/kustomization.yaml
|
||||
- longhorn-ui/kustomization.yaml
|
||||
- postgres/kustomization.yaml
|
||||
- ../platform/vault-csi/kustomization.yaml
|
||||
- ../platform/vault-injector/kustomization.yaml
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/logging/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: logging
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/logging
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
wait: false
|
||||
@ -1,19 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: longhorn-adopt
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
path: ./infrastructure/longhorn/adopt
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
targetNamespace: longhorn-system
|
||||
wait: true
|
||||
@ -4,12 +4,8 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: longhorn-ui
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./infrastructure/longhorn/ui-ingress
|
||||
targetNamespace: longhorn-system
|
||||
prune: true
|
||||
@ -19,5 +15,4 @@ spec:
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: longhorn
|
||||
wait: true
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/longhorn/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: longhorn
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
path: ./infrastructure/longhorn/core
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
targetNamespace: longhorn-system
|
||||
dependsOn:
|
||||
- name: helm
|
||||
- name: longhorn-adopt
|
||||
wait: false
|
||||
@ -1,17 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/maintenance/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: maintenance
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/maintenance
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
wait: false
|
||||
@ -1,18 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/metallb/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: metallb
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 30m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
path: ./infrastructure/metallb
|
||||
prune: true
|
||||
wait: true
|
||||
targetNamespace: metallb-system
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: monitoring
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./services/monitoring
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/postgres/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: postgres
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./infrastructure/postgres
|
||||
prune: true
|
||||
force: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
targetNamespace: postgres
|
||||
dependsOn:
|
||||
- name: longhorn
|
||||
- name: vault
|
||||
- name: vault-csi
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
name: postgres
|
||||
namespace: postgres
|
||||
wait: true
|
||||
@ -1,21 +0,0 @@
|
||||
# clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: resource-guardrails
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
|
||||
spec:
|
||||
interval: 10m
|
||||
suspend: true
|
||||
path: ./infrastructure/resource-guardrails
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
wait: true
|
||||
@ -4,8 +4,6 @@ kind: Kustomization
|
||||
metadata:
|
||||
name: traefik
|
||||
namespace: flux-system
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
interval: 10m
|
||||
path: ./infrastructure/traefik
|
||||
@ -17,5 +15,4 @@ spec:
|
||||
namespace: flux-system
|
||||
dependsOn:
|
||||
- name: core
|
||||
- name: metallb
|
||||
wait: true
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user