Compare commits

..

1 Commits

Author SHA1 Message Date
4901e0352a quality-gate: publish workspace coverage and LOC gauges 2026-04-17 05:37:20 -03:00
660 changed files with 12532 additions and 67009 deletions

17
.gitignore vendored
View File

@ -5,23 +5,6 @@
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
.pytest_cache .pytest_cache
.ruff_cache/
.coverage
build/
test-results/
artifacts/
.venv .venv
.venv-ci .venv-ci
tmp/ tmp/
.mainfix/
.terraform/
**/.terraform/
*.tfvars
*.tfstate
*.tfstate.*
crash.log
terraform/atlas/generated/
# Local demo credentials (never commit)
scripts/ops/hermes_demo.env
scripts/ops/hermes_triage_demo.env

403
Jenkinsfile vendored
View File

@ -7,57 +7,14 @@ pipeline {
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
spec: spec:
serviceAccountName: "jenkins"
nodeSelector: nodeSelector:
hardware: rpi5
kubernetes.io/arch: arm64 kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true" node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-06
- titan-11
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
jenkins/jenkins-jenkins-agent: "true"
containers: containers:
- name: jnlp
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
resources:
requests:
cpu: "25m"
memory: "256Mi"
- name: python - name: python
image: registry.bstein.dev/bstein/python:3.12-slim image: python:3.12-slim
command:
- cat
tty: true
- name: quality-tools
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command: command:
- cat - cat
tty: true tty: true
@ -67,23 +24,9 @@ spec:
environment { environment {
PIP_DISABLE_PIP_VERSION_CHECK = '1' PIP_DISABLE_PIP_VERSION_CHECK = '1'
PYTHONUNBUFFERED = '1' PYTHONUNBUFFERED = '1'
SUITE_NAME = 'titan_iac' SUITE_NAME = 'titan-iac'
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
SONARQUBE_PROJECT_KEY = 'titan_iac'
SONARQUBE_TOKEN = credentials('sonarqube-token')
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
} }
stages { stages {
stage('Checkout') { stage('Checkout') {
@ -93,196 +36,7 @@ spec:
} }
stage('Install deps') { stage('Install deps') {
steps { steps {
sh ''' sh 'pip install --no-cache-dir -r ci/requirements.txt'
set -eu
if ! command -v git >/dev/null 2>&1; then
apt-get update
apt-get install -y --no-install-recommends git ca-certificates
rm -rf /var/lib/apt/lists/*
fi
pip install --no-cache-dir -r ci/requirements.txt
'''
}
}
stage('Prepare local quality evidence') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile local --build-dir build
local_quality_rc=$?
set -e
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
args=(
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
"-Dsonar.login=${SONARQUBE_TOKEN}"
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.sources=."
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
set -e
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
'''
}
sh '''
set -eu
mkdir -p build
python3 - <<'PY'
import base64
import json
import os
import time
import urllib.parse
import urllib.request
from pathlib import Path
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
token = os.getenv('SONARQUBE_TOKEN', '').strip()
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
payload = {
"status": "ERROR",
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
}
if host and project_key:
task_file = Path('.scannerwork/report-task.txt')
task_id = ''
if task_file.exists():
for line in task_file.read_text(encoding='utf-8').splitlines():
key, _, value = line.partition('=')
if key == 'ceTaskId':
task_id = value.strip()
break
if task_id:
ce_query = urllib.parse.urlencode({"id": task_id})
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
ce_request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(ce_request, timeout=12) as response:
ce_payload = json.loads(response.read().decode("utf-8"))
except Exception:
time.sleep(3)
continue
status = str(ce_payload.get("task", {}).get("status", "")).upper()
if status in {"SUCCESS", "FAILED", "CANCELED"}:
break
time.sleep(3)
query = urllib.parse.urlencode({"projectKey": project_key})
request = urllib.request.Request(
f"{host}/api/qualitygates/project_status?{query}",
method="GET",
)
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(request, timeout=12) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as exc: # noqa: BLE001
payload = {"status": "ERROR", "error": str(exc)}
with open(report_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
handle.write("\\n")
PY
'''
}
}
stage('Collect IronBank evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
set +e
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
trivy_rc=$?
set -e
if [ ! -s build/trivy-fs.json ]; then
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
'''
}
sh '''
set -eu
mkdir -p build
if [ -s build/trivy-fs.json ]; then
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
exit 0
fi
python3 - <<'PY'
import json
import os
from pathlib import Path
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
if report_path.exists():
raise SystemExit(0)
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
payload = {
"status": status or "unknown",
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
}
payload = {k: v for k, v in payload.items() if v is not None}
if "status" not in payload:
payload["status"] = "unknown"
payload["note"] = (
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
"or write build/ironbank-compliance.json in image-building repos."
)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
PY
'''
} }
} }
stage('Run quality gate') { stage('Run quality gate') {
@ -312,123 +66,8 @@ PY
stage('Enforce quality gate') { stage('Enforce quality gate') {
steps { steps {
sh ''' sh '''
set -euo pipefail set -eu
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)" test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
fail=0
if [ "${gate_rc}" -ne 0 ]; then
echo "quality gate failed with rc=${gate_rc}" >&2
fail=1
fi
enabled() {
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on) return 0 ;;
*) return 1 ;;
esac
}
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
sonar_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/sonarqube-quality-gate.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${sonar_status}" in
ok|pass|passed|success) ;;
*)
echo "sonarqube gate failed: ${sonar_status}" >&2
fail=1
;;
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1
fi
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
supply_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/ironbank-compliance.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
compliant = payload.get("compliant")
if compliant is True:
print("ok")
elif compliant is False:
print("failed")
else:
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
print(status or "missing")
PY
)"
case "${supply_status}" in
ok|pass|passed|success|compliant) ;;
not_applicable|na|n/a)
if enabled "${ironbank_required}"; then
echo "supply chain gate required but status=${supply_status}" >&2
fail=1
fi
;;
*)
if enabled "${ironbank_required}"; then
echo "supply chain gate failed: ${supply_status}" >&2
fail=1
else
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
fi
;;
esac
fi
exit "${fail}"
''' '''
} }
} }
@ -437,7 +76,7 @@ PY
script { script {
env.FLUX_BRANCH = sh( env.FLUX_BRANCH = sh(
returnStdout: true, returnStdout: true,
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'" script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
).trim() ).trim()
if (!env.FLUX_BRANCH) { if (!env.FLUX_BRANCH) {
error('Flux branch not found in gotk-sync.yaml') error('Flux branch not found in gotk-sync.yaml')
@ -454,22 +93,9 @@ PY
} }
} }
steps { steps {
container('jnlp') {
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) { withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
sh ''' sh '''
set -euo pipefail
if ! command -v git >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then
apk add --no-cache git >/dev/null
elif command -v apt-get >/dev/null 2>&1; then
apt-get update >/dev/null
apt-get install -y git >/dev/null
fi
fi
cd "${WORKSPACE:-$PWD}"
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "workspace is not a git checkout; skipping promote"
exit 0
fi
set +x set +x
git config user.email "jenkins@bstein.dev" git config user.email "jenkins@bstein.dev"
git config user.name "jenkins" git config user.name "jenkins"
@ -480,10 +106,10 @@ PY
} }
} }
} }
}
post { post {
always { always {
script { script {
try {
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) { if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
try { try {
junit allowEmptyResults: true, testResults: 'build/junit-*.xml' junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
@ -491,15 +117,8 @@ PY
echo "junit step unavailable: ${err.class.simpleName}" echo "junit step unavailable: ${err.class.simpleName}"
} }
} }
}
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
} catch (Throwable err) {
if (err.class.simpleName == 'MissingContextVariableException') {
echo 'workspace unavailable; skipping post-build artifact collection'
} else {
throw err
}
}
}
} }
} }
} }

105
Makefile
View File

@ -1,105 +0,0 @@
# Makefile
SHELL := /usr/bin/env bash
KUSTOMIZE ?= kustomize
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
TERRAFORM ?= terraform
TRIVY ?= trivy
BUILD_DIR ?= build
ATLAS_FLUX_ROOT := clusters/atlas/flux-system
AETHER_FLUX_ROOT := clusters/aether/flux-system
AETHER_TF_DIR := terraform/aether
FLUX_KUSTOMIZATION ?= flux-system
FLUX_NAMESPACE ?= flux-system
FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
TRIVY_DB_FLAGS ?= --skip-db-update
.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
help:
@printf '%s\n' \
'Targets:' \
' render Render Atlas, Aether, services, and infrastructure.' \
' render-atlas Render the Atlas Flux root.' \
' render-aether Render the Aether Flux root.' \
' render-services Render every service kustomization.' \
' render-infrastructure Render infrastructure kustomizations.' \
' validate Run render checks plus the local quality gate.' \
' dashboards Regenerate Grafana dashboard artifacts.' \
' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \
' test Run the local quality gate.' \
' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
' flux-inventory Print Flux Kustomization inventory.' \
' trivy-scan Run the local Trivy filesystem scan.' \
' security-report Build IronBank report from build/trivy-fs.json.' \
' security Run Trivy scan and build the IronBank report.' \
' aether-fmt Check Terraform formatting for Aether.' \
' aether-init Initialize Aether Terraform providers.' \
' aether-validate Validate Aether Terraform after init.' \
' aether-plan Plan Aether Terraform from local variables/env.'
render: render-atlas render-aether render-services render-infrastructure
render-atlas:
$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
render-aether:
$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
render-services:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find services -name kustomization.yaml | sort)
render-infrastructure:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find infrastructure -name kustomization.yaml | sort)
validate: render test
dashboards:
$(PYTHON) scripts/render/dashboards_render_atlas.py --build
$(PYTHON) scripts/render/dashboards_render_logs.py --build
$(PYTHON) scripts/render/logging_render_observability.py --build
knowledge:
$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
test:
$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
flux-diff:
flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
flux-inventory:
$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
trivy-scan:
mkdir -p $(BUILD_DIR)
$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
security-report:
@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
security: trivy-scan security-report
aether-fmt:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
aether-init:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
aether-validate:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
aether-plan:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan

View File

@ -12,6 +12,18 @@ This repo contains cluster configuration consumed by Flux:
- service manifests and kustomizations - service manifests and kustomizations
- operational scripts for render/reconcile workflows - operational scripts for render/reconcile workflows
This repo is **not** the Ananke application source repo.
Ananke lives in `bstein/ananke` and orchestrates host-side shutdown/startup behavior around this desired state.
## Validation workflow
```bash
kustomize build services/<app>
kubectl apply --server-side --dry-run=client -k services/<app>
flux reconcile kustomization <name> --namespace flux-system --with-source
```
## Apply model ## Apply model
I use Git + Flux as the source of truth and avoid manual in-cluster edits for durable changes. Use Git + Flux as the source of truth.
Avoid manual in-cluster edits for durable changes.

View File

@ -6,57 +6,14 @@ pipeline {
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
spec: spec:
serviceAccountName: "jenkins"
nodeSelector: nodeSelector:
hardware: rpi5
kubernetes.io/arch: arm64 kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true" node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-06
- titan-11
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
jenkins/jenkins-jenkins-agent: "true"
containers: containers:
- name: jnlp
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
resources:
requests:
cpu: "25m"
memory: "256Mi"
- name: python - name: python
image: registry.bstein.dev/bstein/python:3.12-slim image: python:3.12-slim
command:
- cat
tty: true
- name: quality-tools
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command: command:
- cat - cat
tty: true tty: true
@ -66,23 +23,9 @@ spec:
environment { environment {
PIP_DISABLE_PIP_VERSION_CHECK = '1' PIP_DISABLE_PIP_VERSION_CHECK = '1'
PYTHONUNBUFFERED = '1' PYTHONUNBUFFERED = '1'
SUITE_NAME = 'titan_iac' SUITE_NAME = 'titan-iac'
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
SONARQUBE_PROJECT_KEY = 'titan_iac'
SONARQUBE_TOKEN = credentials('sonarqube-token')
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
} }
stages { stages {
stage('Checkout') { stage('Checkout') {
@ -92,196 +35,7 @@ spec:
} }
stage('Install deps') { stage('Install deps') {
steps { steps {
sh ''' sh 'pip install --no-cache-dir -r ci/requirements.txt'
set -eu
if ! command -v git >/dev/null 2>&1; then
apt-get update
apt-get install -y --no-install-recommends git ca-certificates
rm -rf /var/lib/apt/lists/*
fi
pip install --no-cache-dir -r ci/requirements.txt
'''
}
}
stage('Prepare local quality evidence') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile local --build-dir build
local_quality_rc=$?
set -e
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
args=(
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
"-Dsonar.login=${SONARQUBE_TOKEN}"
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.sources=."
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
set -e
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
'''
}
sh '''
set -eu
mkdir -p build
python3 - <<'PY'
import base64
import json
import os
import time
import urllib.parse
import urllib.request
from pathlib import Path
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
token = os.getenv('SONARQUBE_TOKEN', '').strip()
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
payload = {
"status": "ERROR",
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
}
if host and project_key:
task_file = Path('.scannerwork/report-task.txt')
task_id = ''
if task_file.exists():
for line in task_file.read_text(encoding='utf-8').splitlines():
key, _, value = line.partition('=')
if key == 'ceTaskId':
task_id = value.strip()
break
if task_id:
ce_query = urllib.parse.urlencode({"id": task_id})
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
ce_request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(ce_request, timeout=12) as response:
ce_payload = json.loads(response.read().decode("utf-8"))
except Exception:
time.sleep(3)
continue
status = str(ce_payload.get("task", {}).get("status", "")).upper()
if status in {"SUCCESS", "FAILED", "CANCELED"}:
break
time.sleep(3)
query = urllib.parse.urlencode({"projectKey": project_key})
request = urllib.request.Request(
f"{host}/api/qualitygates/project_status?{query}",
method="GET",
)
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(request, timeout=12) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as exc: # noqa: BLE001
payload = {"status": "ERROR", "error": str(exc)}
with open(report_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
handle.write("\\n")
PY
'''
}
}
stage('Collect IronBank evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
set +e
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
trivy_rc=$?
set -e
if [ ! -s build/trivy-fs.json ]; then
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
'''
}
sh '''
set -eu
mkdir -p build
if [ -s build/trivy-fs.json ]; then
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
exit 0
fi
python3 - <<'PY'
import json
import os
from pathlib import Path
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
if report_path.exists():
raise SystemExit(0)
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
payload = {
"status": status or "unknown",
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
}
payload = {k: v for k, v in payload.items() if v is not None}
if "status" not in payload:
payload["status"] = "unknown"
payload["note"] = (
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
"or write build/ironbank-compliance.json in image-building repos."
)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
PY
'''
} }
} }
stage('Run quality gate') { stage('Run quality gate') {
@ -311,123 +65,8 @@ PY
stage('Enforce quality gate') { stage('Enforce quality gate') {
steps { steps {
sh ''' sh '''
set -euo pipefail set -eu
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)" test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
fail=0
if [ "${gate_rc}" -ne 0 ]; then
echo "quality gate failed with rc=${gate_rc}" >&2
fail=1
fi
enabled() {
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on) return 0 ;;
*) return 1 ;;
esac
}
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
sonar_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/sonarqube-quality-gate.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${sonar_status}" in
ok|pass|passed|success) ;;
*)
echo "sonarqube gate failed: ${sonar_status}" >&2
fail=1
;;
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1
fi
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
supply_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/ironbank-compliance.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
compliant = payload.get("compliant")
if compliant is True:
print("ok")
elif compliant is False:
print("failed")
else:
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
print(status or "missing")
PY
)"
case "${supply_status}" in
ok|pass|passed|success|compliant) ;;
not_applicable|na|n/a)
if enabled "${ironbank_required}"; then
echo "supply chain gate required but status=${supply_status}" >&2
fail=1
fi
;;
*)
if enabled "${ironbank_required}"; then
echo "supply chain gate failed: ${supply_status}" >&2
fail=1
else
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
fi
;;
esac
fi
exit "${fail}"
''' '''
} }
} }
@ -436,7 +75,7 @@ PY
script { script {
env.FLUX_BRANCH = sh( env.FLUX_BRANCH = sh(
returnStdout: true, returnStdout: true,
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'" script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
).trim() ).trim()
if (!env.FLUX_BRANCH) { if (!env.FLUX_BRANCH) {
error('Flux branch not found in gotk-sync.yaml') error('Flux branch not found in gotk-sync.yaml')
@ -453,22 +92,9 @@ PY
} }
} }
steps { steps {
container('jnlp') {
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) { withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
sh ''' sh '''
set -euo pipefail
if ! command -v git >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then
apk add --no-cache git >/dev/null
elif command -v apt-get >/dev/null 2>&1; then
apt-get update >/dev/null
apt-get install -y git >/dev/null
fi
fi
cd "${WORKSPACE:-$PWD}"
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "workspace is not a git checkout; skipping promote"
exit 0
fi
set +x set +x
git config user.email "jenkins@bstein.dev" git config user.email "jenkins@bstein.dev"
git config user.name "jenkins" git config user.name "jenkins"
@ -479,10 +105,10 @@ PY
} }
} }
} }
}
post { post {
always { always {
script { script {
try {
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) { if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
try { try {
junit allowEmptyResults: true, testResults: 'build/junit-*.xml' junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
@ -490,15 +116,8 @@ PY
echo "junit step unavailable: ${err.class.simpleName}" echo "junit step unavailable: ${err.class.simpleName}"
} }
} }
}
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
} catch (Throwable err) {
if (err.class.simpleName == 'MissingContextVariableException') {
echo 'workspace unavailable; skipping post-build artifact collection'
} else {
throw err
}
}
}
} }
} }
} }

View File

@ -6,51 +6,30 @@ from __future__ import annotations
import json import json
import os import os
from glob import glob from glob import glob
from pathlib import Path
import sys
import urllib.error import urllib.error
import urllib.request import urllib.request
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ci.scripts import publish_test_metrics_quality as _quality_helpers
CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
_build_check_statuses = _quality_helpers._build_check_statuses
_combine_statuses = _quality_helpers._combine_statuses
_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
_infer_semgrep_status = _quality_helpers._infer_semgrep_status
_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
_load_optional_json = _quality_helpers._load_optional_json
_normalize_result_status = _quality_helpers._normalize_result_status
def _escape_label(value: str) -> str: def _escape_label(value: str) -> str:
"""Escape a Prometheus label value without changing its content."""
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
def _label_str(labels: dict[str, str]) -> str: def _label_str(labels: dict[str, str]) -> str:
"""Render a stable Prometheus label set from a mapping."""
parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val] parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val]
return "{" + ",".join(parts) + "}" if parts else "" return "{" + ",".join(parts) + "}" if parts else ""
def _read_text(url: str) -> str: def _read_text(url: str) -> str:
"""Fetch a plain-text response body from the given URL."""
with urllib.request.urlopen(url, timeout=10) as response: with urllib.request.urlopen(url, timeout=10) as response:
return response.read().decode("utf-8") return response.read().decode("utf-8")
def _post_text(url: str, payload: str) -> None: def _post_text(url: str, payload: str) -> None:
"""PUT a plain-text payload and fail on any 4xx/5xx response."""
request = urllib.request.Request( request = urllib.request.Request(
url, url,
data=payload.encode("utf-8"), data=payload.encode("utf-8"),
method="PUT", method="POST",
headers={"Content-Type": "text/plain"}, headers={"Content-Type": "text/plain"},
) )
with urllib.request.urlopen(request, timeout=10) as response: with urllib.request.urlopen(request, timeout=10) as response:
@ -59,7 +38,6 @@ def _post_text(url: str, payload: str) -> None:
def _parse_junit(path: str) -> dict[str, int]: def _parse_junit(path: str) -> dict[str, int]:
"""Parse a JUnit XML file into aggregate test counters."""
if not os.path.exists(path): if not os.path.exists(path):
return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
@ -86,7 +64,6 @@ def _parse_junit(path: str) -> dict[str, int]:
def _collect_junit_totals(pattern: str) -> dict[str, int]: def _collect_junit_totals(pattern: str) -> dict[str, int]:
"""Sum JUnit counters across every XML file matching the pattern."""
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
for path in sorted(glob(pattern)): for path in sorted(glob(pattern)):
parsed = _parse_junit(path) parsed = _parse_junit(path)
@ -95,38 +72,7 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]:
return totals return totals
def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
"""Collect individual JUnit test-case statuses for flaky-test trend panels."""
cases: list[tuple[str, str]] = []
for path in sorted(glob(pattern)):
if not os.path.exists(path):
continue
root = ET.parse(path).getroot()
suites: list[ET.Element]
if root.tag == "testsuite":
suites = [root]
elif root.tag == "testsuites":
suites = [elem for elem in root if elem.tag == "testsuite"]
else:
suites = []
for suite in suites:
for test_case in suite.findall("testcase"):
case_name = test_case.attrib.get("name", "").strip()
class_name = test_case.attrib.get("classname", "").strip()
if not case_name:
continue
full_name = f"{class_name}.{case_name}" if class_name else case_name
status = "passed"
if test_case.find("failure") is not None or test_case.find("error") is not None:
status = "failed"
elif test_case.find("skipped") is not None:
status = "skipped"
cases.append((full_name, status))
return cases
def _read_exit_code(path: str) -> int: def _read_exit_code(path: str) -> int:
"""Read the quality-gate exit code, defaulting to failure if missing."""
try: try:
with open(path, "r", encoding="utf-8") as handle: with open(path, "r", encoding="utf-8") as handle:
return int(handle.read().strip()) return int(handle.read().strip())
@ -135,7 +81,6 @@ def _read_exit_code(path: str) -> int:
def _load_summary(path: str) -> dict: def _load_summary(path: str) -> dict:
"""Load the JSON quality-gate summary, returning an empty mapping on error."""
try: try:
with open(path, "r", encoding="utf-8") as handle: with open(path, "r", encoding="utf-8") as handle:
return json.load(handle) return json.load(handle)
@ -144,7 +89,6 @@ def _load_summary(path: str) -> dict:
def _summary_float(summary: dict, key: str) -> float: def _summary_float(summary: dict, key: str) -> float:
"""Extract a float-like value from the summary, defaulting to 0.0."""
value = summary.get(key) value = summary.get(key)
if isinstance(value, (int, float)): if isinstance(value, (int, float)):
return float(value) return float(value)
@ -152,7 +96,6 @@ def _summary_float(summary: dict, key: str) -> float:
def _summary_int(summary: dict, key: str) -> int: def _summary_int(summary: dict, key: str) -> int:
"""Extract an int-like value from the summary, defaulting to 0."""
value = summary.get(key) value = summary.get(key)
if isinstance(value, int): if isinstance(value, int):
return value return value
@ -162,7 +105,6 @@ def _summary_int(summary: dict, key: str) -> int:
def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float: def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float:
"""Return the current counter value for a labeled metric if present."""
text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics") text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics")
for line in text.splitlines(): for line in text.splitlines():
if not line.startswith(metric + "{"): if not line.startswith(metric + "{"):
@ -183,34 +125,22 @@ def _build_payload(
suite: str, suite: str,
status: str, status: str,
tests: dict[str, int], tests: dict[str, int],
test_cases: list[tuple[str, str]],
ok_count: int, ok_count: int,
failed_count: int, failed_count: int,
branch: str, branch: str,
build_number: str, build_number: str,
jenkins_job: str,
summary: dict | None = None, summary: dict | None = None,
workspace_line_coverage_percent: float = 0.0, workspace_line_coverage_percent: float = 0.0,
source_files_total: int = 0,
source_lines_over_500: int = 0, source_lines_over_500: int = 0,
check_statuses: dict[str, str] | None = None,
) -> str: ) -> str:
"""Build the Pushgateway payload for the current suite run."""
passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0) passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0)
build_labels = _label_str( build_labels = _label_str(
{ {
"suite": suite, "suite": suite,
"branch": branch or "unknown", "branch": branch or "unknown",
"build_number": build_number or "unknown", "build_number": build_number or "unknown",
"jenkins_job": jenkins_job or suite,
} }
) )
test_case_base_labels = {
"suite": suite,
"branch": branch or "unknown",
"build_number": build_number or "unknown",
"jenkins_job": jenkins_job or suite,
}
lines = [ lines = [
"# TYPE platform_quality_gate_runs_total counter", "# TYPE platform_quality_gate_runs_total counter",
f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}', f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
@ -223,87 +153,43 @@ def _build_payload(
"# TYPE titan_iac_quality_gate_run_status gauge", "# TYPE titan_iac_quality_gate_run_status gauge",
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}', f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}',
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}', f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}',
"# TYPE platform_quality_gate_build_info gauge",
f"platform_quality_gate_build_info{build_labels} 1",
"# TYPE titan_iac_quality_gate_build_info gauge", "# TYPE titan_iac_quality_gate_build_info gauge",
f"titan_iac_quality_gate_build_info{build_labels} 1", f"titan_iac_quality_gate_build_info{build_labels} 1",
"# TYPE platform_quality_gate_workspace_line_coverage_percent gauge", "# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}', f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}',
"# TYPE platform_quality_gate_source_files_total gauge",
f'platform_quality_gate_source_files_total{{suite="{suite}"}} {source_files_total}',
"# TYPE platform_quality_gate_source_lines_over_500_total gauge", "# TYPE platform_quality_gate_source_lines_over_500_total gauge",
f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}', f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
] ]
if check_statuses: results = summary.get("results", []) if isinstance(summary, dict) else []
if results:
lines.append("# TYPE titan_iac_quality_gate_checks_total gauge") lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
for check_name in CANONICAL_CHECKS: for result in results:
check_status = check_statuses.get(check_name, "not_applicable") check_name = result.get("name")
check_status = result.get("status")
if not check_name or not check_status:
continue
lines.append( lines.append(
f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1' f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(str(check_name))}",result="{_escape_label(str(check_status))}"}} 1'
)
lines.append("# TYPE platform_quality_gate_test_case_result gauge")
if test_cases:
for test_name, test_status in test_cases:
labels = {
**test_case_base_labels,
"test": test_name,
"status": test_status,
}
lines.append(
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
)
else:
labels = {**test_case_base_labels, "test": "__no_test_cases__", "status": "skipped"}
lines.append(
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
) )
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"
def main() -> int: def main() -> int:
"""Publish the quality-gate metrics and print a compact run summary.""" suite = os.getenv("SUITE_NAME", "titan-iac")
suite = os.getenv("SUITE_NAME", "titan_iac")
pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091") pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci") job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml")) junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc")) exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc"))
summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json") summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json")
branch = os.getenv("BRANCH_NAME") or os.getenv("GIT_BRANCH") or "unknown" branch = os.getenv("BRANCH_NAME", os.getenv("GIT_BRANCH", ""))
if branch.startswith("origin/"):
branch = branch[len("origin/") :]
build_number = os.getenv("BUILD_NUMBER", "") build_number = os.getenv("BUILD_NUMBER", "")
jenkins_job = os.getenv("JOB_NAME", "titan-iac")
tests = _collect_junit_totals(junit_glob) tests = _collect_junit_totals(junit_glob)
test_cases = _collect_junit_cases(junit_glob)
exit_code = _read_exit_code(exit_code_path) exit_code = _read_exit_code(exit_code_path)
status = "ok" if exit_code == 0 else "failed" status = "ok" if exit_code == 0 else "failed"
summary = _load_summary(summary_path) summary = _load_summary(summary_path)
workspace_line_coverage_percent = _summary_float(summary, "workspace_line_coverage_percent") workspace_line_coverage_percent = _summary_float(summary, "workspace_line_coverage_percent")
if workspace_line_coverage_percent <= 0:
workspace_line_coverage_percent = _infer_workspace_coverage_percent(summary, "build/coverage-unit.xml")
source_files_total = _summary_int(summary, "source_files_total")
source_lines_over_500 = _summary_int(summary, "source_lines_over_500") source_lines_over_500 = _summary_int(summary, "source_lines_over_500")
if source_lines_over_500 <= 0:
source_lines_over_500 = _infer_source_lines_over_500(summary)
sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
truthy = {"1", "true", "yes", "on"}
supply_chain_required = (
os.getenv("QUALITY_GATE_IRONBANK_REQUIRED", "0").strip().lower() in truthy
or os.getenv("PUBLISH_IMAGES", "false").strip().lower() in truthy
)
check_statuses = _build_check_statuses(
summary=summary,
tests=tests,
workspace_line_coverage_percent=workspace_line_coverage_percent,
source_lines_over_500=source_lines_over_500,
sonarqube_report=sonarqube_report,
semgrep_report=semgrep_report,
supply_chain_report=supply_chain_report,
supply_chain_required=supply_chain_required,
)
ok_count = int( ok_count = int(
_fetch_existing_counter( _fetch_existing_counter(
@ -328,17 +214,13 @@ def main() -> int:
suite=suite, suite=suite,
status=status, status=status,
tests=tests, tests=tests,
test_cases=test_cases,
ok_count=ok_count, ok_count=ok_count,
failed_count=failed_count, failed_count=failed_count,
branch=branch, branch=branch,
build_number=build_number, build_number=build_number,
jenkins_job=jenkins_job,
summary=summary, summary=summary,
workspace_line_coverage_percent=workspace_line_coverage_percent, workspace_line_coverage_percent=workspace_line_coverage_percent,
source_files_total=source_files_total,
source_lines_over_500=source_lines_over_500, source_lines_over_500=source_lines_over_500,
check_statuses=check_statuses,
) )
push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}" push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}"
_post_text(push_url, payload) _post_text(push_url, payload)
@ -352,14 +234,13 @@ def main() -> int:
"tests_skipped": tests["skipped"], "tests_skipped": tests["skipped"],
"ok_count": ok_count, "ok_count": ok_count,
"failed_count": failed_count, "failed_count": failed_count,
"checks_recorded": len(check_statuses), "checks_recorded": len(summary.get("results", [])) if isinstance(summary, dict) else 0,
"workspace_line_coverage_percent": workspace_line_coverage_percent, "workspace_line_coverage_percent": workspace_line_coverage_percent,
"source_files_total": source_files_total,
"source_lines_over_500": source_lines_over_500, "source_lines_over_500": source_lines_over_500,
} }
print(json.dumps(summary, sort_keys=True)) print(json.dumps(summary, sort_keys=True))
return 0 return 0
if __name__ == "__main__": # pragma: no cover if __name__ == "__main__":
raise SystemExit(main()) raise SystemExit(main())

View File

@ -1,221 +0,0 @@
#!/usr/bin/env python3
"""Quality/status helpers for publish_test_metrics."""
from __future__ import annotations
import json
from pathlib import Path
import xml.etree.ElementTree as ET
SUCCESS_STATUSES = {"ok", "pass", "passed", "success", "compliant"}
NOT_APPLICABLE_STATUSES = {"not_applicable", "n/a", "na", "none", "skipped"}
FAILED_STATUSES = {"failed", "fail", "error", "errors", "warn", "warning", "red"}
CANONICAL_CHECKS = [
"tests",
"coverage",
"loc",
"docs_naming",
"gate_glue",
"sonarqube",
"semgrep",
"supply_chain",
]
def _infer_workspace_coverage_percent(summary: dict, default_xml: str) -> float:
"""Infer workspace line coverage from quality summary coverage XML metadata."""
results = summary.get("results", []) if isinstance(summary, dict) else []
coverage_xml = default_xml
for result in results:
if not isinstance(result, dict):
continue
if str(result.get("name") or "").strip().lower() != "coverage":
continue
candidate = str(result.get("coverage_xml") or "").strip()
if candidate:
coverage_xml = candidate
break
xml_path = Path(coverage_xml)
if not xml_path.exists():
return 0.0
try:
root = ET.parse(xml_path).getroot()
line_rate = root.attrib.get("line-rate")
if line_rate is None:
return 0.0
return float(line_rate) * 100.0
except (ET.ParseError, OSError, ValueError):
return 0.0
def _infer_source_lines_over_500(summary: dict) -> int:
"""Infer over-limit source file count from hygiene issue payloads."""
results = summary.get("results", []) if isinstance(summary, dict) else []
for result in results:
if not isinstance(result, dict):
continue
if str(result.get("name") or "").strip().lower() not in {"hygiene", "loc", "smell"}:
continue
issues = result.get("issues")
if not isinstance(issues, list):
continue
return sum(1 for item in issues if isinstance(item, str) and item.startswith("file exceeds"))
return 0
def _normalize_result_status(value: str | None, default: str = "failed") -> str:
"""Map arbitrary check status text into canonical check result buckets."""
if not value:
return default
normalized = value.strip().lower()
if normalized in SUCCESS_STATUSES:
return "ok"
if normalized in NOT_APPLICABLE_STATUSES:
return "not_applicable"
if normalized in FAILED_STATUSES:
return "failed"
return default
def _load_optional_json(path: str | None) -> dict:
"""Load an optional JSON report file, returning an empty object when absent."""
if not path:
return {}
candidate = Path(path)
if not candidate.exists():
return {}
try:
return json.loads(candidate.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
def _combine_statuses(statuses: list[str]) -> str:
"""Roll up many check statuses into one canonical result."""
if not statuses:
return "not_applicable"
if any(status == "failed" for status in statuses):
return "failed"
if all(status == "not_applicable" for status in statuses):
return "not_applicable"
if all(status in {"ok", "not_applicable"} for status in statuses):
return "ok"
return "failed"
def _infer_sonarqube_status(report: dict) -> str:
"""Infer canonical SonarQube check status from its JSON report payload."""
if not report:
return "not_applicable"
status = (
report.get("projectStatus", {}).get("status")
or report.get("qualityGate", {}).get("status")
or report.get("status")
)
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _infer_supply_chain_status(report: dict, required: bool) -> str:
"""Infer canonical supply-chain status from IronBank/artifact report payload."""
if not report:
return "failed" if required else "not_applicable"
compliant = report.get("compliant")
if isinstance(compliant, bool):
if compliant:
return "ok"
return "failed" if required else "not_applicable"
status = report.get("status")
if status is None:
return "failed" if required else "not_applicable"
normalized = _normalize_result_status(str(status), default="failed")
if normalized == "failed" and not required:
return "not_applicable"
if normalized == "not_applicable" and required:
return "failed"
return normalized
def _infer_semgrep_status(report: dict) -> str:
"""Infer canonical Semgrep check status from its JSON report payload."""
if not report:
return "not_applicable"
status = report.get("status")
if status is None:
blocking_findings = report.get("blocking_findings")
errors_total = report.get("errors_total")
if isinstance(blocking_findings, int) and isinstance(errors_total, int):
return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _build_check_statuses(
summary: dict | None,
tests: dict[str, int],
workspace_line_coverage_percent: float,
source_lines_over_500: int,
sonarqube_report: dict,
semgrep_report: dict,
supply_chain_report: dict,
supply_chain_required: bool,
) -> dict[str, str]:
"""Generate the canonical quality-check status map for dashboarding."""
raw_results = summary.get("results", []) if isinstance(summary, dict) else []
status_by_name: dict[str, str] = {}
for result in raw_results:
if not isinstance(result, dict):
continue
check_name = str(result.get("name") or "").strip().lower()
if not check_name:
continue
status_by_name[check_name] = _normalize_result_status(result.get("status"), default="failed")
tests_status = status_by_name.get("tests")
if not tests_status:
candidate_keys = ["unit", "integration", "e2e", "pytest", "test", "tests"]
candidates = [status_by_name[key] for key in candidate_keys if key in status_by_name]
if candidates:
tests_status = _combine_statuses(candidates)
elif tests["tests"] > 0:
tests_status = "ok" if (tests["failures"] + tests["errors"]) == 0 else "failed"
else:
tests_status = "not_applicable"
coverage_status = status_by_name.get("coverage")
if not coverage_status:
if workspace_line_coverage_percent > 0:
coverage_status = "ok" if workspace_line_coverage_percent >= 95.0 else "failed"
else:
coverage_status = "not_applicable"
loc_status = status_by_name.get("loc")
if not loc_status:
loc_status = "ok" if source_lines_over_500 == 0 else "failed"
docs_naming_status = status_by_name.get("docs_naming")
if not docs_naming_status:
candidates = [status_by_name[key] for key in ["docs", "hygiene", "smell", "lint", "naming"] if key in status_by_name]
docs_naming_status = _combine_statuses(candidates) if candidates else "not_applicable"
gate_glue_status = status_by_name.get("gate_glue")
if not gate_glue_status:
candidates = [status_by_name[key] for key in ["gate_glue", "glue", "gate"] if key in status_by_name]
gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
supply_chain_report,
required=supply_chain_required,
)
return {
"tests": tests_status,
"coverage": coverage_status,
"loc": loc_status,
"docs_naming": docs_naming_status,
"gate_glue": gate_glue_status,
"sonarqube": sonarqube_status,
"semgrep": semgrep_status,
"supply_chain": supply_chain_status,
}

View File

@ -1,159 +0,0 @@
#!/usr/bin/env python3
"""Build a compact Semgrep gate report from Semgrep JSON output."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
BLOCKING_SEVERITIES = {"ERROR"}
SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
SONAR_SEVERITY_BY_SEMGREP = {
"ERROR": "CRITICAL",
"WARNING": "MAJOR",
"INFO": "MINOR",
"UNKNOWN": "INFO",
}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk, returning an error-shaped payload on failure."""
if not path.exists():
return {"errors": [{"message": f"report missing: {path}"}], "results": []}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
if not isinstance(payload, dict):
return {"errors": [{"message": "report payload is not an object"}], "results": []}
return payload
def _finding_severity(finding: dict[str, Any]) -> str:
"""Return a normalized Semgrep finding severity."""
extra = finding.get("extra")
severity = extra.get("severity") if isinstance(extra, dict) else None
normalized = str(severity or "UNKNOWN").strip().upper()
return normalized if normalized in SEVERITIES else "UNKNOWN"
def build_report(
semgrep_payload: dict[str, Any],
*,
semgrep_exit_code: int,
blocking_severities: set[str] | None = None,
) -> dict[str, Any]:
"""Summarize Semgrep evidence into the quality-gate report contract."""
blocking = blocking_severities or BLOCKING_SEVERITIES
raw_results = semgrep_payload.get("results", [])
raw_errors = semgrep_payload.get("errors", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = {severity: 0 for severity in SEVERITIES}
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)
severity_counts[severity] += 1
if severity in blocking:
blocking_findings += 1
engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
status = "failed" if engine_error or blocking_findings else "ok"
return {
"status": status,
"scanner": "semgrep",
"semgrep_rc": semgrep_exit_code,
"findings_total": len(results),
"blocking_findings": blocking_findings,
"errors_total": len(errors),
"severity_counts": severity_counts,
"blocking_severities": sorted(blocking),
}
def _line_number(value: Any, default: int = 1) -> int:
"""Return a Sonar-compatible one-indexed line number."""
try:
line = int(value)
except (TypeError, ValueError):
return default
return max(line, default)
def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
"""Convert one Semgrep finding into SonarQube generic issue format."""
path = str(finding.get("path") or "").strip()
if not path:
return None
extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
severity = _finding_severity(finding)
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
if end_line < start_line:
end_line = start_line
return {
"engineId": "semgrep",
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
"type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
"severity": SONAR_SEVERITY_BY_SEMGREP[severity],
"primaryLocation": {
"message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
"filePath": path,
"textRange": {
"startLine": start_line,
"endLine": end_line,
},
},
}
def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"""Build SonarQube generic external issues from Semgrep JSON output."""
raw_results = semgrep_payload.get("results", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
return {"issues": issues}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after Semgrep finishes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--semgrep-json", required=True)
parser.add_argument("--exit-code", type=int, default=0)
parser.add_argument("--output", required=True)
parser.add_argument("--sonar-issues-output")
parser.add_argument(
"--blocking-severity",
action="append",
default=[],
help="Severity that should mark the report failed. Defaults to ERROR.",
)
args = parser.parse_args(argv)
blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
payload = _read_json(Path(args.semgrep_json))
report = build_report(
payload,
semgrep_exit_code=args.exit_code,
blocking_severities=blocking or BLOCKING_SEVERITIES,
)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.sonar_issues_output:
sonar_output = Path(args.sonar_issues_output)
sonar_output.parent.mkdir(parents=True, exist_ok=True)
sonar_output.write_text(
json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -1,177 +0,0 @@
"""Build a titan-iac supply-chain compliance report from Trivy evidence."""
from __future__ import annotations
import argparse
import datetime as dt
import json
from pathlib import Path
from typing import Any
FAIL_SEVERITIES = {"HIGH", "CRITICAL"}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk for use as pipeline evidence."""
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"{path} must contain a JSON object")
return payload
def _parse_day(raw: str | None) -> dt.date | None:
"""Parse an ISO day while letting optional waiver dates stay optional."""
if not raw:
return None
return dt.date.fromisoformat(raw)
def _today(override: str | None = None) -> dt.date:
"""Return the policy day so tests can pin expiry behavior."""
return _parse_day(override) or dt.date.today()
def _load_waiver_pairs(path: Path | None, policy_day: dt.date) -> tuple[set[tuple[str, str]], int]:
"""Return active ``(misconfiguration id, target)`` waivers and expired count."""
if path is None or not path.exists():
return set(), 0
payload = _read_json(path)
default_expires_at = payload.get("default_expires_at")
active: set[tuple[str, str]] = set()
expired = 0
for entry in payload.get("misconfigurations", []):
if not isinstance(entry, dict):
continue
misconfiguration_id = str(entry.get("id") or "").strip()
if not misconfiguration_id:
continue
expires_at = _parse_day(str(entry.get("expires_at") or default_expires_at or ""))
targets = entry.get("targets", [])
if not isinstance(targets, list):
continue
if expires_at and expires_at < policy_day:
expired += len(targets)
continue
# Waivers are target-specific so a new unsafe manifest fails until it is
# either fixed or deliberately accepted with a fresh expiration.
for target in targets:
if isinstance(target, str) and target:
active.add((misconfiguration_id, target))
return active, expired
def _iter_failed_misconfigurations(payload: dict[str, Any]):
"""Yield failed high/critical Trivy misconfiguration records."""
for result in payload.get("Results", []):
if not isinstance(result, dict):
continue
target = str(result.get("Target") or "")
for item in result.get("Misconfigurations") or []:
if not isinstance(item, dict):
continue
if item.get("Status") != "FAIL":
continue
if str(item.get("Severity") or "").upper() not in FAIL_SEVERITIES:
continue
yield target, item
def _count_vulnerabilities(payload: dict[str, Any], severity: str) -> int:
"""Count Trivy vulnerabilities at a specific severity."""
count = 0
for result in payload.get("Results", []):
if not isinstance(result, dict):
continue
for item in result.get("Vulnerabilities") or []:
if isinstance(item, dict) and str(item.get("Severity") or "").upper() == severity:
count += 1
return count
def _count_secrets(payload: dict[str, Any]) -> int:
"""Count detected secrets in the Trivy filesystem report."""
count = 0
for result in payload.get("Results", []):
if isinstance(result, dict):
count += len(result.get("Secrets") or [])
return count
def build_report(
trivy_payload: dict[str, Any],
waiver_path: Path | None = None,
today_override: str | None = None,
) -> dict[str, Any]:
"""Build the compliance summary consumed by the quality gate."""
policy_day = _today(today_override)
active_waivers, expired_waivers = _load_waiver_pairs(waiver_path, policy_day)
open_misconfigs: list[dict[str, str]] = []
waived_misconfigs = 0
for target, item in _iter_failed_misconfigurations(trivy_payload):
misconfiguration_id = str(item.get("ID") or "")
if (misconfiguration_id, target) in active_waivers:
waived_misconfigs += 1
continue
open_misconfigs.append(
{
"id": misconfiguration_id,
"target": target,
"severity": str(item.get("Severity") or ""),
"title": str(item.get("Title") or ""),
}
)
critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
high = _count_vulnerabilities(trivy_payload, "HIGH")
secrets = _count_secrets(trivy_payload)
status = (
"ok"
if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
else "failed"
)
return {
"status": status,
"compliant": status == "ok",
"category": "artifact_security",
"scan_type": "filesystem",
"scanner": "trivy",
"critical_vulnerabilities": critical,
"high_vulnerabilities": high,
"high_vulnerability_policy": "observe",
"secrets": secrets,
"high_or_critical_misconfigurations": len(open_misconfigs),
"waived_misconfigurations": waived_misconfigs,
"expired_waivers": expired_waivers,
"waiver_file": str(waiver_path) if waiver_path else "",
"open_misconfiguration_examples": open_misconfigs[:20],
}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after the Trivy scan completes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--trivy-json", required=True)
parser.add_argument("--waivers")
parser.add_argument("--output", required=True)
parser.add_argument("--today")
args = parser.parse_args(argv)
trivy_payload = _read_json(Path(args.trivy_json))
waiver_path = Path(args.waivers) if args.waivers else None
report = build_report(trivy_payload, waiver_path=waiver_path, today_override=args.today)
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -1,34 +0,0 @@
"""Skip live glue checks when the VictoriaMetrics endpoint is unavailable."""
from __future__ import annotations
import os
import socket
from urllib.parse import urlparse
import pytest
def _endpoint_available() -> bool:
"""Return whether the configured VictoriaMetrics host resolves locally."""
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
parsed = urlparse(vm_url)
host = parsed.hostname
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if not host:
return False
try:
socket.getaddrinfo(host, port)
except socket.gaierror:
return False
return True
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
"""Mark glue tests skipped when the VictoriaMetrics host cannot resolve."""
del config
if _endpoint_available():
return
skip_marker = pytest.mark.skip(reason="VictoriaMetrics endpoint is not reachable in this workspace")
for item in items:
item.add_marker(skip_marker)

View File

@ -1,108 +0,0 @@
"""Glue checks for Ariadne schedules exported to VictoriaMetrics."""
from __future__ import annotations
import os
from datetime import datetime, timezone
from pathlib import Path
import requests
import yaml
CONFIG_PATH = Path(__file__).with_name("config.yaml")
def _load_config() -> dict:
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
return yaml.safe_load(handle) or {}
def _query(promql: str) -> list[dict]:
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
response = requests.get(f"{vm_url}/api/v1/query", params={"query": promql}, timeout=10)
response.raise_for_status()
payload = response.json()
return payload.get("data", {}).get("result", [])
def _expected_tasks() -> list[dict]:
cfg = _load_config()
tasks = [
_normalize_task(item, cfg)
for item in cfg.get("ariadne_schedule_tasks", [])
]
assert tasks, "No Ariadne schedule tasks configured"
return tasks
def _normalize_task(item: object, cfg: dict) -> dict:
if isinstance(item, str):
return {
"task": item,
"check_last_success": True,
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
}
if isinstance(item, dict):
normalized = dict(item)
normalized.setdefault("check_last_success", True)
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
return normalized
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
tracked = [item for item in tasks if item.get("check_last_success")]
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
return tracked
def _task_regex(tasks: list[dict]) -> str:
return "|".join(item["task"] for item in tasks)
def test_ariadne_schedule_series_exist():
tasks = _expected_tasks()
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing next-run metrics for: {', '.join(missing)}"
def test_ariadne_schedule_recent_success():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing last-success metrics for: {', '.join(missing)}"
now = datetime.now(timezone.utc)
age_by_task = {
item.get("metric", {}).get("task"): (now - datetime.fromtimestamp(float(item["value"][1]), tz=timezone.utc)).total_seconds() / 3600
for item in series
}
too_old = [
f"{task} ({age_by_task[task]:.1f}h > {item['max_success_age_hours']}h)"
for item in tasks
if (task := item["task"]) in age_by_task and age_by_task[task] > float(item["max_success_age_hours"])
]
assert not too_old, "Ariadne schedules are stale: " + ", ".join(too_old)
def test_ariadne_schedule_last_status_present_and_boolean():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing last-status metrics for: {', '.join(missing)}"
invalid = []
for item in series:
task = item.get("metric", {}).get("task")
value = float(item["value"][1])
if value not in (0.0, 1.0):
invalid.append(f"{task}={value}")
assert not invalid, f"Unexpected Ariadne last-status values: {', '.join(invalid)}"

View File

@ -1,5 +1,3 @@
"""Glue checks for the metrics the quality-gate publishes."""
from __future__ import annotations from __future__ import annotations
import os import os
@ -25,63 +23,26 @@ def _query(promql: str) -> list[dict]:
return payload.get("data", {}).get("result", []) return payload.get("data", {}).get("result", [])
def _expected_tasks() -> list[dict]: def test_glue_metrics_present():
cfg = _load_config() series = _query('kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}')
tasks = [ assert series, "No glue cronjob label series found"
_normalize_task(item, cfg)
for item in cfg.get("ariadne_schedule_tasks", [])
]
assert tasks, "No Ariadne schedule tasks configured"
return tasks
def _normalize_task(item: object, cfg: dict) -> dict: def test_glue_metrics_success_join():
if isinstance(item, str): query = (
return { "kube_cronjob_status_last_successful_time "
"task": item, 'and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}'
"check_last_success": True, )
"max_success_age_hours": cfg.get("max_success_age_hours", 48), series = _query(query)
} assert series, "No glue cronjob last success series found"
if isinstance(item, dict):
normalized = dict(item)
normalized.setdefault("check_last_success", True)
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
return normalized
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
tracked = [item for item in tasks if item.get("check_last_success")]
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
return tracked
def _task_regex(tasks: list[dict]) -> str:
return "|".join(item["task"] for item in tasks)
def test_ariadne_schedule_metrics_present(): def test_ariadne_schedule_metrics_present():
tasks = _expected_tasks() cfg = _load_config()
selector = _task_regex(tasks) expected = cfg.get("ariadne_schedule_tasks", [])
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}') if not expected:
seen = {item.get("metric", {}).get("task") for item in series} return
missing = [item["task"] for item in tasks if item["task"] not in seen] series = _query("ariadne_schedule_next_run_timestamp_seconds")
tasks = {item.get("metric", {}).get("task") for item in series}
missing = [task for task in expected if task not in tasks]
assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}" assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}"
def test_ariadne_schedule_success_and_status_metrics_present():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
success = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
status = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
success_tasks = {item.get("metric", {}).get("task") for item in success}
status_tasks = {item.get("metric", {}).get("task") for item in status}
expected = {item["task"] for item in tasks}
missing_success = sorted(expected - success_tasks)
missing_status = sorted(expected - status_tasks)
assert not missing_success, f"Missing Ariadne success metrics for: {', '.join(missing_success)}"
assert not missing_status, f"Missing Ariadne status metrics for: {', '.join(missing_status)}"

View File

@ -1,405 +0,0 @@
{
"version": 1,
"generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
"default_expires_at": "2026-09-30",
"ticket": "atlas-quality-wave-k8s-hardening",
"default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
"misconfigurations": [
{
"id": "DS-0002",
"targets": [
"dockerfiles/Dockerfile.ananke-node-helper"
]
},
{
"id": "KSV-0009",
"targets": [
"services/mailu/vip-controller.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml"
]
},
{
"id": "KSV-0010",
"targets": [
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
},
{
"id": "KSV-0014",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
"infrastructure/core/ntp-sync-daemonset.yaml",
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"infrastructure/postgres/statefulset.yaml",
"infrastructure/vault-csi/vault-csi-provider.yaml",
"services/ai-llm/deployment.yaml",
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
"services/comms/coturn.yaml",
"services/comms/element-call-deployment.yaml",
"services/comms/guest-name-job.yaml",
"services/comms/guest-register-deployment.yaml",
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
"services/comms/vault-sync-deployment.yaml",
"services/comms/wellknown.yaml",
"services/crypto/monerod/deployment.yaml",
"services/crypto/wallet-monero-temp/deployment.yaml",
"services/crypto/xmr-miner/deployment.yaml",
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
"services/finance/actual-budget-deployment.yaml",
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
"services/health/wger-deployment.yaml",
"services/health/wger-user-sync-cronjob.yaml",
"services/jellyfin/deployment.yaml",
"services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
"services/monitoring/vmalert-atlas-availability.yaml",
"services/monitoring/vault-sync-deployment.yaml",
"services/nextcloud-mail-sync/cronjob.yaml",
"services/nextcloud/collabora.yaml",
"services/nextcloud/cronjob.yaml",
"services/nextcloud/deployment.yaml",
"services/nextcloud/maintenance-cronjob.yaml",
"services/oauth2-proxy/deployment.yaml",
"services/openldap/statefulset.yaml",
"services/outline/deployment.yaml",
"services/outline/redis-deployment.yaml",
"services/pegasus/deployment.yaml",
"services/pegasus/vault-sync-deployment.yaml",
"services/planka/deployment.yaml",
"services/quality/oauth2-proxy-sonarqube.yaml",
"services/quality/sonarqube-deployment.yaml",
"services/quality/sonarqube-exporter-deployment.yaml",
"services/sui-metrics/base/deployment.yaml",
"services/typhon/vault-sync-deployment.yaml",
"services/vault/k8s-auth-config-cronjob.yaml",
"services/vault/oidc-config-cronjob.yaml",
"services/vault/statefulset.yaml",
"services/vaultwarden/deployment.yaml"
]
},
{
"id": "KSV-0017",
"targets": [
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
},
{
"id": "KSV-0041",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"infrastructure/traefik/clusterrole.yaml",
"services/bstein-dev-home/rbac.yaml",
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/comms/mas-secrets-ensure-rbac.yaml",
"services/maintenance/apps/soteria-rbac.yaml"
]
},
{
"id": "KSV-0047",
"targets": [
"services/monitoring/rbac.yaml"
]
},
{
"id": "KSV-0053",
"targets": [
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/apps/ariadne-rbac.yaml"
]
},
{
"id": "KSV-0056",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml",
"services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml"
]
},
{
"id": "KSV-0114",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml"
]
},
{
"id": "KSV-0118",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
"infrastructure/core/coredns-deployment.yaml",
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
"infrastructure/core/ntp-sync-daemonset.yaml",
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"infrastructure/postgres/statefulset.yaml",
"infrastructure/vault-csi/vault-csi-provider.yaml",
"services/ai-llm/deployment.yaml",
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
"services/comms/coturn.yaml",
"services/comms/element-call-deployment.yaml",
"services/comms/guest-name-job.yaml",
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
"services/comms/vault-sync-deployment.yaml",
"services/comms/wellknown.yaml",
"services/crypto/monerod/deployment.yaml",
"services/crypto/wallet-monero-temp/deployment.yaml",
"services/crypto/xmr-miner/deployment.yaml",
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
"services/health/wger-deployment.yaml",
"services/health/wger-user-sync-cronjob.yaml",
"services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
"services/monitoring/vmalert-atlas-availability.yaml",
"services/monitoring/vault-sync-deployment.yaml",
"services/nextcloud/collabora.yaml",
"services/oauth2-proxy/deployment.yaml",
"services/openldap/statefulset.yaml",
"services/outline/deployment.yaml",
"services/outline/redis-deployment.yaml",
"services/pegasus/vault-sync-deployment.yaml",
"services/quality/oauth2-proxy-sonarqube.yaml",
"services/quality/sonarqube-deployment.yaml",
"services/quality/sonarqube-exporter-deployment.yaml",
"services/sui-metrics/base/deployment.yaml",
"services/sui-metrics/overlays/atlas/patch-node-selector.yaml",
"services/typhon/deployment.yaml",
"services/typhon/vault-sync-deployment.yaml",
"services/vault/k8s-auth-config-cronjob.yaml",
"services/vault/oidc-config-cronjob.yaml",
"services/vaultwarden/deployment.yaml"
]
},
{
"id": "KSV-0121",
"targets": [
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml"
]
}
]
}

View File

@ -1,6 +0,0 @@
# clusters/aether/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- platform
- applications

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: ai-llm name: ai-llm
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec: spec:
interval: 10m interval: 10m
suspend: false
timeout: 30m
path: ./services/ai-llm path: ./services/ai-llm
targetNamespace: ai targetNamespace: ai
prune: true prune: true

View File

@ -4,12 +4,9 @@ kind: Kustomization
metadata: metadata:
name: bstein-dev-home-migrations name: bstein-dev-home-migrations
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
spec: spec:
interval: 10m interval: 10m
path: ./services/bstein-dev-home/migration-jobs path: ./services/bstein-dev-home/oneoffs/migrations
prune: true prune: true
force: true force: true
sourceRef: sourceRef:

View File

@ -13,14 +13,14 @@ spec:
git: git:
checkout: checkout:
ref: ref:
branch: main branch: feature/ariadne
commit: commit:
author: author:
email: ops@bstein.dev email: ops@bstein.dev
name: flux-bot name: flux-bot
messageTemplate: "chore(bstein-dev-home): automated image update" messageTemplate: "chore(bstein-dev-home): automated image update"
push: push:
branch: main branch: feature/ariadne
update: update:
strategy: Setters strategy: Setters
path: services/bstein-dev-home path: services/bstein-dev-home

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: bstein-dev-home name: bstein-dev-home
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/bstein-dev-home path: ./services/bstein-dev-home

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra-auth/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra-auth
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra-auth
targetNamespace: sso
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: keycloak
- name: vault
- name: vault-injector
wait: false
timeout: 10m

View File

@ -1,29 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra/image-automation.yaml
# Staged for the first Cassandra image rollout. Add this file to the parent
# applications kustomization after the namespace exists and the Harbor repos
# have initial tags.
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: cassandra
namespace: cassandra
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(cassandra): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/cassandra

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra
targetNamespace: cassandra
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: traefik
- name: vault
- name: vault-csi
- name: vault-injector
wait: false
timeout: 20m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: comms name: comms
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
prune: true prune: true
@ -15,3 +13,5 @@ spec:
path: ./services/comms path: ./services/comms
targetNamespace: comms targetNamespace: comms
timeout: 2m timeout: 2m
dependsOn:
- name: traefik

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: crypto name: crypto
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/crypto path: ./services/crypto

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: finance name: finance
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/finance path: ./services/finance
prune: true prune: true
sourceRef: sourceRef:

View File

@ -1,31 +0,0 @@
# clusters/atlas/flux-system/applications/game-stream/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: game-stream
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
spec:
interval: 10m
suspend: true
path: ./services/game-stream
targetNamespace: game-stream
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: keycloak
- name: traefik
- name: vault
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-wolf
namespace: game-stream
wait: false
timeout: 10m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: gitea name: gitea
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/gitea path: ./services/gitea

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: harbor name: harbor
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/harbor path: ./services/harbor

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: health name: health
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/health path: ./services/health
prune: true prune: true
sourceRef: sourceRef:
@ -19,6 +15,7 @@ spec:
dependsOn: dependsOn:
- name: keycloak - name: keycloak
- name: postgres - name: postgres
- name: traefik
- name: vault - name: vault
healthChecks: healthChecks:
- apiVersion: apps/v1 - apiVersion: apps/v1

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-chat
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-chat
targetNamespace: hermes-chat
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
dependsOn:
- name: core
- name: hermes
- name: longhorn

View File

@ -1,16 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-triage-demo
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-triage-demo
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
timeout: 5m

View File

@ -1,62 +0,0 @@
# clusters/atlas/flux-system/applications/hermes/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
path: ./services/hermes
targetNamespace: hermes
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 45m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-model-gate
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-ollama
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-agent
namespace: hermes
- apiVersion: apps/v1
kind: StatefulSet
name: hermes-chat-tenant
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-chat-router
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes-agent
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes-chat
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes-triage
namespace: hermes
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: vault

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: jellyfin name: jellyfin
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/jellyfin path: ./services/jellyfin
targetNamespace: jellyfin targetNamespace: jellyfin
prune: true prune: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: jenkins name: jenkins
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/jenkins path: ./services/jenkins
prune: true prune: true
sourceRef: sourceRef:
@ -18,6 +14,7 @@ spec:
targetNamespace: jenkins targetNamespace: jenkins
dependsOn: dependsOn:
- name: helm - name: helm
- name: traefik
healthChecks: healthChecks:
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: keycloak name: keycloak
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
prune: true prune: true

View File

@ -21,20 +21,10 @@ resources:
- sui-metrics/kustomization.yaml - sui-metrics/kustomization.yaml
- openldap/kustomization.yaml - openldap/kustomization.yaml
- keycloak/kustomization.yaml - keycloak/kustomization.yaml
- quality/kustomization.yaml
- oauth2-proxy/kustomization.yaml - oauth2-proxy/kustomization.yaml
- mailu/kustomization.yaml - mailu/kustomization.yaml
- jenkins/kustomization.yaml - jenkins/kustomization.yaml
- ai-llm/kustomization.yaml - ai-llm/kustomization.yaml
- openclaw/kustomization.yaml
- hermes/kustomization.yaml
- hermes-chat/kustomization.yaml
- hermes-triage-demo/kustomization.yaml
- game-stream/kustomization.yaml
- cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml
- cassandra/image-automation.yaml
- veles/kustomization.yaml
- typhon/kustomization.yaml - typhon/kustomization.yaml
- nextcloud/kustomization.yaml - nextcloud/kustomization.yaml
- nextcloud-mail-sync/kustomization.yaml - nextcloud-mail-sync/kustomization.yaml

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: mailu name: mailu
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
spec: spec:
interval: 10m interval: 10m
suspend: true
sourceRef: sourceRef:
kind: GitRepository kind: GitRepository
name: flux-system name: flux-system

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: monerod name: monerod
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/crypto/monerod path: ./services/crypto/monerod

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: nextcloud-mail-sync name: nextcloud-mail-sync
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
spec: spec:
interval: 10m interval: 10m
suspend: true
prune: true prune: true
sourceRef: sourceRef:
kind: GitRepository kind: GitRepository

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: nextcloud name: nextcloud
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/nextcloud path: ./services/nextcloud
targetNamespace: nextcloud targetNamespace: nextcloud
prune: true prune: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: oauth2-proxy name: oauth2-proxy
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
prune: true prune: true

View File

@ -1,22 +0,0 @@
# clusters/atlas/flux-system/applications/openclaw/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: openclaw
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
path: ./services/openclaw
targetNamespace: openclaw
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: core
- name: longhorn

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: openldap name: openldap
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
prune: true prune: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: outline name: outline
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/outline path: ./services/outline
prune: true prune: true
sourceRef: sourceRef:
@ -19,6 +15,7 @@ spec:
dependsOn: dependsOn:
- name: keycloak - name: keycloak
- name: mailu - name: mailu
- name: traefik
healthChecks: healthChecks:
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: pegasus name: pegasus
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/pegasus path: ./services/pegasus

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: planka name: planka
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/planka path: ./services/planka
prune: true prune: true
sourceRef: sourceRef:
@ -19,6 +15,7 @@ spec:
dependsOn: dependsOn:
- name: keycloak - name: keycloak
- name: mailu - name: mailu
- name: traefik
healthChecks: healthChecks:
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment

View File

@ -1,38 +0,0 @@
# clusters/atlas/flux-system/applications/quality/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: quality
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
spec:
interval: 10m
suspend: true
path: ./services/quality
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: quality
dependsOn:
- name: cert-manager
- name: keycloak
- name: vault
- name: postgres
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: sonarqube
namespace: quality
- apiVersion: apps/v1
kind: Deployment
name: sonarqube-exporter
namespace: quality
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-sonarqube
namespace: quality
wait: false
timeout: 20m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: sui-metrics name: sui-metrics
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/sui-metrics/overlays/atlas path: ./services/sui-metrics/overlays/atlas

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: typhon name: typhon
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/typhon path: ./services/typhon
prune: true prune: true
sourceRef: sourceRef:

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: vault name: vault
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
sourceRef: sourceRef:

View File

@ -4,12 +4,9 @@ kind: Kustomization
metadata: metadata:
name: vaultwarden name: vaultwarden
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: false
sourceRef: sourceRef:
kind: GitRepository kind: GitRepository
name: flux-system name: flux-system
@ -20,3 +17,4 @@ spec:
wait: true wait: true
dependsOn: dependsOn:
- name: helm - name: helm
- name: traefik

View File

@ -1,29 +0,0 @@
# clusters/atlas/flux-system/applications/veles/image-automation.yaml
# Staged for the first Veles image rollout. Add this file to the parent
# applications kustomization after the namespace exists and the Harbor repos
# have initial tags.
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: veles
namespace: veles
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(veles): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/veles

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/veles/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: veles
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/veles
targetNamespace: veles
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: traefik
- name: vault
- name: vault-csi
- name: vault-injector
wait: false
timeout: 20m

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: wallet-monero-temp name: wallet-monero-temp
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/crypto/wallet-monero-temp path: ./services/crypto/wallet-monero-temp
targetNamespace: crypto targetNamespace: crypto
prune: true prune: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: xmr-miner name: xmr-miner
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./services/crypto/xmr-miner path: ./services/crypto/xmr-miner
targetNamespace: crypto targetNamespace: crypto
prune: true prune: true

View File

@ -5966,9 +5966,6 @@ spec:
- args: - args:
- --events-addr=http://notification-controller.$(RUNTIME_NAMESPACE).svc.cluster.local./ - --events-addr=http://notification-controller.$(RUNTIME_NAMESPACE).svc.cluster.local./
- --watch-all-namespaces=true - --watch-all-namespaces=true
- --concurrent=4
- --requeue-dependency=5s
- --interval-jitter-percentage=30
- --log-level=info - --log-level=info
- --log-encoding=json - --log-encoding=json
- --enable-leader-election - --enable-leader-election

View File

@ -7,7 +7,7 @@ metadata:
name: flux-system name: flux-system
namespace: flux-system namespace: flux-system
spec: spec:
interval: 15m0s interval: 1m0s
ref: ref:
branch: main branch: main
secretRef: secretRef:
@ -20,7 +20,7 @@ metadata:
name: flux-system name: flux-system
namespace: flux-system namespace: flux-system
spec: spec:
interval: 1h0m0s interval: 10m0s
path: ./clusters/atlas/flux-system path: ./clusters/atlas/flux-system
prune: true prune: true
sourceRef: sourceRef:

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: cert-manager-cleanup name: cert-manager-cleanup
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
path: ./infrastructure/cert-manager/cleanup path: ./infrastructure/cert-manager/cleanup

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: cert-manager name: cert-manager
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
path: ./infrastructure/cert-manager path: ./infrastructure/cert-manager

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: core name: core
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./infrastructure/core path: ./infrastructure/core

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/platform/descheduler/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: descheduler
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
spec:
interval: 30m
suspend: true
path: ./infrastructure/descheduler
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: kube-system
dependsOn:
- name: helm
- name: core
wait: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: gitops-ui name: gitops-ui
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
spec: spec:
interval: 10m interval: 10m
suspend: true
timeout: 10m timeout: 10m
path: ./services/gitops-ui path: ./services/gitops-ui
prune: true prune: true
@ -20,4 +16,5 @@ spec:
targetNamespace: flux-system targetNamespace: flux-system
dependsOn: dependsOn:
- name: helm - name: helm
- name: traefik
wait: true wait: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: helm name: helm
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
sourceRef: sourceRef:

View File

@ -4,8 +4,6 @@ kind: Kustomization
resources: resources:
- core/kustomization.yaml - core/kustomization.yaml
- helm/kustomization.yaml - helm/kustomization.yaml
- descheduler/kustomization.yaml
- resource-guardrails/kustomization.yaml
- cert-manager/kustomization.yaml - cert-manager/kustomization.yaml
- metallb/kustomization.yaml - metallb/kustomization.yaml
- traefik/kustomization.yaml - traefik/kustomization.yaml

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: logging name: logging
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/logging path: ./services/logging

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: longhorn-adopt name: longhorn-adopt
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
path: ./infrastructure/longhorn/adopt path: ./infrastructure/longhorn/adopt

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata: metadata:
name: longhorn-ui name: longhorn-ui
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
spec: spec:
interval: 10m interval: 10m
suspend: true
path: ./infrastructure/longhorn/ui-ingress path: ./infrastructure/longhorn/ui-ingress
targetNamespace: longhorn-system targetNamespace: longhorn-system
prune: true prune: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: longhorn name: longhorn
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
path: ./infrastructure/longhorn/core path: ./infrastructure/longhorn/core

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: maintenance name: maintenance
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/maintenance path: ./services/maintenance

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: metallb name: metallb
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
sourceRef: sourceRef:

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: monitoring name: monitoring
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./services/monitoring path: ./services/monitoring

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: postgres name: postgres
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./infrastructure/postgres path: ./infrastructure/postgres

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: resource-guardrails
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
spec:
interval: 10m
suspend: true
path: ./infrastructure/resource-guardrails
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: core
wait: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: traefik name: traefik
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 10m interval: 10m
path: ./infrastructure/traefik path: ./infrastructure/traefik

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: vault-csi name: vault-csi
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
sourceRef: sourceRef:

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata: metadata:
name: vault-injector name: vault-injector
namespace: flux-system namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec: spec:
interval: 30m interval: 30m
path: ./infrastructure/vault-injector path: ./infrastructure/vault-injector

View File

@ -1,4 +1,4 @@
# clusters/aether/flux-system/platform/kustomization.yaml # clusters/oceanus/applications/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: [] resources: []

View File

@ -0,0 +1,9 @@
# clusters/oceanus/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Populate when oceanus cluster is bootstrapped with Flux.
# - gotk-components.yaml
# - gotk-sync.yaml
- ../platform
- ../applications

View File

@ -0,0 +1,6 @@
# clusters/oceanus/platform/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../infrastructure/modules/base
- ../../infrastructure/modules/profiles/oceanus-validator

View File

@ -2,8 +2,4 @@ FROM python:3.11-slim
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
RUN pip install --no-cache-dir requests psycopg2-binary \ RUN pip install --no-cache-dir requests psycopg2-binary
&& groupadd --system guest-tools \
&& useradd --system --uid 65532 --gid guest-tools --home-dir /nonexistent --shell /usr/sbin/nologin guest-tools
USER guest-tools

View File

@ -1,12 +1,15 @@
# Use the mirrored Harbor artifact so CI does not depend on Docker Hub egress. FROM --platform=$BUILDPLATFORM opensearchproject/data-prepper:2.8.0 AS source
FROM registry.bstein.dev/streaming/data-prepper@sha256:32ac6ad42e0f12da08bebee307e290b17d127b30def9b06eeaffbcbbc5033e83
FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre
ENV DATA_PREPPER_PATH=/usr/share/data-prepper ENV DATA_PREPPER_PATH=/usr/share/data-prepper
USER root RUN useradd -u 10001 -M -U -d / -s /usr/sbin/nologin data_prepper \
RUN apt-get update \ && mkdir -p /var/log/data-prepper
&& apt-get install -y --no-install-recommends bc \
&& rm -rf /var/lib/apt/lists/* COPY --from=source /usr/share/data-prepper /usr/share/data-prepper
RUN chown -R 10001:10001 /usr/share/data-prepper /var/log/data-prepper
USER 10001 USER 10001
WORKDIR /usr/share/data-prepper WORKDIR /usr/share/data-prepper

View File

@ -1,334 +0,0 @@
# syntax=docker/dockerfile:1
# dockerfiles/Dockerfile.hermes-agent
FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
USER root
# Keep a credential-free search provider available for private chat tenants.
# Paid/provider-backed search remains selectable through normal Hermes config.
RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
# Keep dashboard chat sockets tied to the intended React mount and conversation.
# A resumed conversation needs a different PTY attachment key from a fresh chat;
# reconnects to that same conversation must keep using the same key.
RUN node <<'NODE'
const fs = require("node:fs");
const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
let source = fs.readFileSync(path, "utf8");
const socketBefore = [
' const url = await api.buildWsUrl("/api/pty", params);',
' const ws = new WebSocket(url);',
].join("\n");
const socketAfter = [
' const url = await api.buildWsUrl("/api/pty", params);',
' if (unmounting) return;',
' const ws = new WebSocket(url);',
].join("\n");
const attachBefore = ' params.attach = ptyAttachToken(forceFresh);';
const attachAfter = [
' const attachScope = resumeParam',
' ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
' : `fresh:${scopedProfile ?? ""}`;',
' params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
].join("\n");
if (!source.includes(socketBefore)) {
throw new Error("Hermes ChatPage WebSocket patch context changed");
}
if (!source.includes(attachBefore)) {
throw new Error("Hermes ChatPage PTY attachment patch context changed");
}
source = source.replace(socketBefore, socketAfter);
source = source.replace(attachBefore, attachAfter);
fs.writeFileSync(path, source);
NODE
# The upstream OIDC gate authenticates users but deliberately treats the
# dashboard as one shared workstation. Allow a deployment to narrow that
# workstation to explicit OIDC subjects. Enforce this after normal provider
# verification so a denied account is a 403, not a misleading provider 503.
RUN python - <<'PY'
from pathlib import Path
path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
source = path.read_text()
helper_before = '''def _client_ip(request: Request) -> str:
fwd = request.headers.get("x-forwarded-for", "")
if fwd:
return fwd.split(",")[0].strip()
return request.client.host if request.client else ""
'''
helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
"""Apply an optional deployment-level OIDC-subject allowlist."""
import os
allowed = {
value.strip()
for value in os.environ.get(
"HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
).split(",")
if value.strip()
}
return not allowed or session.user_id in allowed
def _user_forbidden_response() -> Response:
"""Return an authorization failure without exposing identities."""
return JSONResponse(
{
"error": "forbidden",
"detail": "This Atlas account is not authorized for this dashboard.",
},
status_code=403,
)
'''
refresh_before = ''' new_session, refreshing_provider = refreshed
request.state.session = new_session
response = await call_next(request)
'''
refresh_after = ''' new_session, refreshing_provider = refreshed
if not _dashboard_user_allowed(new_session):
return _user_forbidden_response()
request.state.session = new_session
response = await call_next(request)
'''
final_before = ''' request.state.session = session
return await call_next(request)
'''
final_after = ''' if not _dashboard_user_allowed(session):
return _user_forbidden_response()
request.state.session = session
return await call_next(request)
'''
for before, after, label in (
(helper_before, helper_after, "allowlist helper"),
(refresh_before, refresh_after, "refreshed session"),
(final_before, final_after, "verified session"),
):
if before not in source:
raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
source = source.replace(before, after, 1)
path.write_text(source)
PY
# Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
# restoration but before Hermes builds its provider-specific system prompt.
RUN python - <<'PY'
from pathlib import Path
plugins_path = Path("/opt/hermes/hermes_cli/plugins.py")
plugins = plugins_path.read_text()
hooks_before = ''' "pre_llm_call",
"post_llm_call",
'''
hooks_after = ''' "pre_llm_call",
"pre_turn_route",
"post_llm_call",
'''
if plugins.count(hooks_before) != 1:
raise SystemExit(
"Hermes pre-turn hook registry context changed: expected 1, "
f"found {plugins.count(hooks_before)}"
)
plugins_path.write_text(plugins.replace(hooks_before, hooks_after, 1))
turn_path = Path("/opt/hermes/agent/turn_context.py")
turn = turn_path.read_text()
turn_before = ''' agent._restore_primary_runtime()
'''
turn_after = turn_before + '''
# Trusted coordinator plugins may select the provider/model/effort for this
# turn. Run this before system-prompt restoration so the prompt and runtime
# always describe the same selected provider.
try:
from hermes_cli.plugins import has_hook, invoke_hook
if has_hook("pre_turn_route"):
invoke_hook(
"pre_turn_route",
agent=agent,
user_message=user_message,
conversation_history=list(conversation_history or []),
session_id=agent.session_id or "",
platform=agent.platform or "",
)
except Exception:
logger.warning("pre_turn_route hook failed", exc_info=True)
'''
if turn.count(turn_before) != 1:
raise SystemExit(
"Hermes pre-turn routing context changed: expected 1, "
f"found {turn.count(turn_before)}"
)
turn_path.write_text(turn.replace(turn_before, turn_after, 1))
PY
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
# Upstream currently applies only statically declared model_routes there, so
# the UI can display one model while the gateway silently runs another. Honor
# trusted first-party provider selections and cap all chat reasoning at xhigh.
RUN python - <<'PY'
from pathlib import Path
path = Path("/opt/hermes/gateway/platforms/api_server.py")
source = path.read_text()
route_before = ''' def _resolve_route(self, model_alias: Any) -> Optional[Dict[str, Any]]:
"""Return the model_routes entry for *model_alias*, or None."""
if not self._model_routes or not isinstance(model_alias, str):
return None
return self._model_routes.get(model_alias)
'''
route_after = route_before + ''' def _resolve_request_route(self, body: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Resolve a static route or a trusted WebUI provider/model selection."""
route = self._resolve_route(body.get("model"))
if route is not None:
return route
provider = body.get("provider")
model = body.get("model")
allowed_providers = {"openai-codex", "anthropic"}
if provider not in allowed_providers or not isinstance(model, str):
return None
model = model.strip()
if not model or len(model) > 128 or any(
char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:/+-"
for char in model
):
return None
return {"provider": provider, "model": model}
'''
signature_before = ''' gateway_session_key: Optional[str] = None,
route: Optional[Dict[str, Any]] = None,
) -> Any:
'''
signature_after = ''' gateway_session_key: Optional[str] = None,
route: Optional[Dict[str, Any]] = None,
reasoning_effort: Any = None,
) -> Any:
'''
reasoning_before = ''' runtime_kwargs = _resolve_runtime_agent_kwargs()
reasoning_config = GatewayRunner._load_reasoning_config()
model = _resolve_gateway_model()
'''
reasoning_after = ''' runtime_kwargs = _resolve_runtime_agent_kwargs()
reasoning_config = GatewayRunner._load_reasoning_config()
from hermes_constants import parse_reasoning_effort
requested_reasoning = parse_reasoning_effort(reasoning_effort)
if requested_reasoning is not None:
reasoning_config = requested_reasoning
if reasoning_config and reasoning_config.get("effort") == "max":
reasoning_config = {"enabled": True, "effort": "xhigh"}
model = _resolve_gateway_model()
'''
runs_route_before = ''' # Per-client model routing for /v1/runs (see model_routes).
route = self._resolve_route(body.get("model"))
'''
runs_route_after = ''' # Honor both static routes and the WebUI's trusted provider/model pick.
route = self._resolve_request_route(body)
'''
runs_agent_before = ''' gateway_session_key=gateway_session_key,
route=route,
)
'''
runs_agent_after = ''' gateway_session_key=gateway_session_key,
route=route,
reasoning_effort=body.get("reasoning_effort"),
)
'''
for before, after, label, count in (
(route_before, route_after, "request route resolver", 1),
(signature_before, signature_after, "agent reasoning argument", 1),
(reasoning_before, reasoning_after, "reasoning clamp", 1),
(runs_route_before, runs_route_after, "runs route", 1),
):
if source.count(before) != count:
raise SystemExit(
f"Hermes API {label} patch context changed: expected {count}, "
f"found {source.count(before)}"
)
source = source.replace(before, after, count)
# The same argument tail appears in other handlers. Restrict replacement to
# the /v1/runs section so non-WebUI API surfaces retain upstream behavior.
runs_start = source.index(" async def _handle_runs(")
runs_source = source[runs_start:]
if runs_source.count(runs_agent_before) != 1:
raise SystemExit(
"Hermes API /v1/runs agent-call patch context changed: expected 1, "
f"found {runs_source.count(runs_agent_before)}"
)
runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
source = source[:runs_start] + runs_source
path.write_text(source)
PY
COPY dockerfiles/hermes-python-sandbox-tool.py /opt/hermes/tools/python_sandbox_tool.py
COPY dockerfiles/hermes-public-extract/__init__.py /opt/hermes/plugins/web/public_extract/__init__.py
COPY dockerfiles/hermes-public-extract/plugin.yaml /opt/hermes/plugins/web/public_extract/plugin.yaml
COPY dockerfiles/hermes-public-extract/provider.py /opt/hermes/plugins/web/public_extract/provider.py
# Per-capability custom backends are resolved before upstream discovers web
# plugins, causing extract_backend to fall through to the shared search-only
# backend. Discover bundled plugins before checking a custom capability name.
RUN python - <<'PY'
from pathlib import Path
path = Path("/opt/hermes/tools/web_tools.py")
source = path.read_text()
before = ''' cfg = _load_web_config()
specific = (cfg.get(f"{capability}_backend") or "").lower().strip()
if specific and _is_backend_available(specific):
return specific
'''
after = ''' cfg = _load_web_config()
specific = (cfg.get(f"{capability}_backend") or "").lower().strip()
if specific and specific not in _LEGACY_WEB_BACKENDS:
_ensure_web_plugins_loaded()
if specific and _is_backend_available(specific):
return specific
'''
if source.count(before) != 1:
raise SystemExit(
"Hermes custom web capability patch context changed: expected 1, "
f"found {source.count(before)}"
)
path.write_text(source.replace(before, after, 1))
PY
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
RUN cd /opt/hermes/web \
&& npm run build \
&& grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
&& grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
&& grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
/opt/hermes/hermes_cli/dashboard_auth/middleware.py \
&& grep -Fq '_resolve_request_route' \
/opt/hermes/gateway/platforms/api_server.py \
&& grep -Fq 'reasoning_effort=body.get("reasoning_effort")' \
/opt/hermes/gateway/platforms/api_server.py \
&& grep -Fq '"pre_turn_route"' /opt/hermes/hermes_cli/plugins.py \
&& grep -Fq 'invoke_hook(' /opt/hermes/agent/turn_context.py \
&& /opt/hermes/.venv/bin/python -m py_compile \
/opt/hermes/gateway/platforms/api_server.py \
/opt/hermes/agent/turn_context.py \
/opt/hermes/tools/web_tools.py \
/opt/hermes/tools/python_sandbox_tool.py \
/opt/hermes/plugins/web/public_extract/provider.py \
&& /opt/hermes/.venv/bin/python -c 'import ddgs' \
&& chmod 0755 /opt/hermes/bin/hermes-session-migrate

View File

@ -1,23 +0,0 @@
# dockerfiles/Dockerfile.hermes-chat-router
FROM --platform=$BUILDPLATFORM golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 AS build
ARG TARGETOS
ARG TARGETARCH
WORKDIR /src
COPY services/hermes/router/ ./
RUN GO111MODULE=off CGO_ENABLED=0 go test . \
&& GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=off CGO_ENABLED=0 go build \
-trimpath \
-ldflags="-s -w" \
-o /out/chat-router .
FROM scratch
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build --chown=10000:10000 /out/chat-router /chat-router
USER 10000:10000
EXPOSE 8080
ENTRYPOINT ["/chat-router"]

View File

@ -1,18 +0,0 @@
# syntax=docker/dockerfile:1
# dockerfiles/Dockerfile.hermes-chat-sandbox
FROM python:3.13-slim@sha256:9662417aace5ae7b8e2609cce472b72a8958e134ba372808abe9cc1a0c0125e6
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
HOME=/workspace
RUN groupadd --gid 20000 sandbox \
&& useradd --uid 20000 --gid 20000 --home-dir /workspace --no-create-home sandbox
COPY --chown=20000:20000 dockerfiles/hermes-chat-sandbox-server.py /opt/sandbox/server.py
USER 20000:20000
WORKDIR /workspace
EXPOSE 9080
ENTRYPOINT ["python", "-I", "/opt/sandbox/server.py"]

View File

@ -1,75 +0,0 @@
# syntax=docker/dockerfile:1
# dockerfiles/Dockerfile.hermes-webui
FROM ghcr.io/nesquena/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2 AS webui
FROM registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
USER root
# Keep WebUI and Hermes pinned together. The WebUI imports Hermes internals,
# while the gateway remains the only process that owns an agent conversation.
COPY --from=webui /apptoo /opt/hermes-webui
# The account policy caps user-selected reasoning at xhigh even when a provider
# advertises a newer, more expensive level.
RUN /opt/hermes/.venv/bin/python - <<'PY'
from pathlib import Path
config = Path("/opt/hermes-webui/api/config.py")
source = config.read_text(encoding="utf-8")
before = 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh", "max")'
after = 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")'
if before not in source:
raise SystemExit("Hermes WebUI reasoning-effort patch context changed")
config.write_text(source.replace(before, after, 1), encoding="utf-8")
index = Path("/opt/hermes-webui/static/index.html")
source = index.read_text(encoding="utf-8")
before = ' <div class="reasoning-option" data-effort="max">Max</div>\n'
if before not in source:
raise SystemExit("Hermes WebUI xhigh UI patch context changed")
index.write_text(source.replace(before, "", 1), encoding="utf-8")
PY
RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \
&& grep -Fq 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")' \
/opt/hermes-webui/api/config.py \
&& ! grep -Fq 'data-effort="max"' /opt/hermes-webui/static/index.html
# Exercise the real server process in the target architecture before publish.
RUN set -eu; \
mkdir -p /tmp/hermes-webui-smoke/home /tmp/hermes-webui-smoke/state /tmp/hermes-webui-smoke/workspace; \
HERMES_HOME=/tmp/hermes-webui-smoke/home \
HOME=/tmp/hermes-webui-smoke/home \
HERMES_WEBUI_STATE_DIR=/tmp/hermes-webui-smoke/state \
HERMES_WEBUI_DEFAULT_WORKSPACE=/tmp/hermes-webui-smoke/workspace \
HERMES_WEBUI_HOST=127.0.0.1 \
HERMES_WEBUI_PORT=18787 \
HERMES_WEBUI_SKIP_ONBOARDING=1 \
/opt/hermes/.venv/bin/python /opt/hermes-webui/server.py >/tmp/hermes-webui-smoke.log 2>&1 & \
server_pid=$!; \
ready=0; \
for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do \
if /opt/hermes/.venv/bin/python -c 'from urllib.request import urlopen; urlopen("http://127.0.0.1:18787/health", timeout=2).read()' >/dev/null 2>&1; then ready=1; break; fi; \
sleep 1; \
done; \
kill "${server_pid}" 2>/dev/null || true; \
wait "${server_pid}" 2>/dev/null || true; \
if [ "${ready}" != "1" ]; then cat /tmp/hermes-webui-smoke.log; exit 1; fi; \
rm -rf /tmp/hermes-webui-smoke /tmp/hermes-webui-smoke.log
ENV HERMES_WEBUI_AGENT_DIR=/opt/hermes \
HERMES_WEBUI_HOST=0.0.0.0 \
HERMES_WEBUI_PORT=8787 \
HERMES_WEBUI_CHAT_BACKEND=gateway \
HERMES_WEBUI_GATEWAY_BASE_URL=http://127.0.0.1:8642 \
HERMES_WEBUI_GATEWAY_USE_RUNS_API=true \
HERMES_WEBUI_SKIP_ONBOARDING=1 \
HERMES_WEBUI_SECURE=1 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /opt/hermes-webui
USER 10000:10000
EXPOSE 8787
ENTRYPOINT ["/opt/hermes/.venv/bin/python", "/opt/hermes-webui/server.py"]

View File

@ -1,13 +1,10 @@
FROM ghcr.io/element-hq/lk-jwt-service:0.3.0 AS base FROM ghcr.io/element-hq/lk-jwt-service:0.3.0 AS base
FROM alpine:3.20 FROM alpine:3.20
RUN apk add --no-cache ca-certificates \ RUN apk add --no-cache ca-certificates
&& addgroup -S livekit-token \
&& adduser -S -D -H -u 65532 -G livekit-token livekit-token
COPY --from=base /lk-jwt-service /lk-jwt-service COPY --from=base /lk-jwt-service /lk-jwt-service
COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
RUN chmod 0755 /entrypoint.sh RUN chmod 0755 /entrypoint.sh
USER livekit-token
ENTRYPOINT ["/entrypoint.sh"] ENTRYPOINT ["/entrypoint.sh"]
CMD ["/lk-jwt-service"] CMD ["/lk-jwt-service"]

View File

@ -29,12 +29,10 @@ FROM ${DEBIAN_IMAGE}
RUN set -eux; \ RUN set -eux; \
apt-get update; \ apt-get update; \
apt-get install -y --no-install-recommends ca-certificates; \ apt-get install -y --no-install-recommends ca-certificates; \
update-ca-certificates; rm -rf /var/lib/apt/lists/*; \ update-ca-certificates; rm -rf /var/lib/apt/lists/*
groupadd --system p2pool; \
useradd --system --uid 65532 --gid p2pool --home-dir /nonexistent --shell /usr/sbin/nologin p2pool
COPY --from=fetch /out/p2pool /usr/local/bin/p2pool COPY --from=fetch /out/p2pool /usr/local/bin/p2pool
RUN /usr/local/bin/p2pool --version || true RUN /usr/local/bin/p2pool --version || true
EXPOSE 3333 EXPOSE 3333
USER p2pool
ENTRYPOINT ["/usr/local/bin/p2pool"] ENTRYPOINT ["/usr/local/bin/p2pool"]

View File

@ -26,12 +26,9 @@ RUN set -eux; \
curl -fsSL "$URL" -o /opt/monero/monero.tar.bz2; \ curl -fsSL "$URL" -o /opt/monero/monero.tar.bz2; \
tar -xjf /opt/monero/monero.tar.bz2 -C /opt/monero --strip-components=1; \ tar -xjf /opt/monero/monero.tar.bz2 -C /opt/monero --strip-components=1; \
install -m 0755 /opt/monero/monero-wallet-rpc /usr/local/bin/monero-wallet-rpc; \ install -m 0755 /opt/monero/monero-wallet-rpc /usr/local/bin/monero-wallet-rpc; \
rm -f /opt/monero/monero.tar.bz2; \ rm -f /opt/monero/monero.tar.bz2
groupadd --system monero; \
useradd --system --uid 1000 --gid monero --home-dir /nonexistent --shell /usr/sbin/nologin monero
ENV PATH="/usr/local/bin:/usr/bin:/bin" ENV PATH="/usr/local/bin:/usr/bin:/bin"
RUN /usr/local/bin/monero-wallet-rpc --version || true RUN /usr/local/bin/monero-wallet-rpc --version || true
EXPOSE 18083 EXPOSE 18083
USER monero

View File

@ -23,14 +23,10 @@ RUN set -eux; \
mkdir -p /opt/monero; \ mkdir -p /opt/monero; \
tar -xjf /tmp/monero.tar.bz2 -C /opt/monero --strip-components=1; \ tar -xjf /tmp/monero.tar.bz2 -C /opt/monero --strip-components=1; \
rm -f /tmp/monero.tar.bz2; \ rm -f /tmp/monero.tar.bz2; \
groupadd --system monero; \
useradd --system --uid 1000 --gid monero --home-dir /nonexistent --shell /usr/sbin/nologin monero; \
mkdir -p /data; \ mkdir -p /data; \
chown monero:monero /data; \
chmod 0770 /data chmod 0770 /data
ENV LD_LIBRARY_PATH=/opt/monero:/opt/monero/lib \ ENV LD_LIBRARY_PATH=/opt/monero:/opt/monero/lib \
PATH="/opt/monero:${PATH}" PATH="/opt/monero:${PATH}"
USER monero
CMD ["/opt/monero/monerod", "--version"] CMD ["/opt/monero/monerod", "--version"]

View File

@ -1,13 +1,10 @@
FROM quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 AS base FROM quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 AS base
FROM alpine:3.20 FROM alpine:3.20
RUN apk add --no-cache ca-certificates \ RUN apk add --no-cache ca-certificates
&& addgroup -S oauth2-proxy \
&& adduser -S -D -H -u 65532 -G oauth2-proxy oauth2-proxy
COPY --from=base /bin/oauth2-proxy /bin/oauth2-proxy COPY --from=base /bin/oauth2-proxy /bin/oauth2-proxy
COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
RUN chmod 0755 /entrypoint.sh RUN chmod 0755 /entrypoint.sh
USER oauth2-proxy
ENTRYPOINT ["/entrypoint.sh"] ENTRYPOINT ["/entrypoint.sh"]
CMD ["/bin/oauth2-proxy"] CMD ["/bin/oauth2-proxy"]

View File

@ -1,13 +1,10 @@
FROM registry.bstein.dev/streaming/pegasus:1.2.32 AS base FROM registry.bstein.dev/streaming/pegasus:1.2.32 AS base
FROM alpine:3.20 FROM alpine:3.20
RUN apk add --no-cache ca-certificates \ RUN apk add --no-cache ca-certificates
&& addgroup -S pegasus \
&& adduser -S -D -H -u 65532 -G pegasus pegasus
COPY --from=base /pegasus /pegasus COPY --from=base /pegasus /pegasus
COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
RUN chmod 0755 /entrypoint.sh RUN chmod 0755 /entrypoint.sh
USER pegasus
ENTRYPOINT ["/entrypoint.sh"] ENTRYPOINT ["/entrypoint.sh"]
CMD ["/pegasus"] CMD ["/pegasus"]

View File

@ -1,48 +0,0 @@
# dockerfiles/Dockerfile.quality-tools
FROM debian:bookworm-slim
ARG SONAR_SCANNER_VERSION=8.0.1.6346
ARG TRIVY_VERSION=0.70.0
ENV TRIVY_CACHE_DIR=/opt/trivy-cache
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
git \
jq \
unzip \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --system quality-tools \
&& useradd --system --uid 65532 --gid quality-tools --home-dir /nonexistent --shell /usr/sbin/nologin quality-tools
RUN set -eux; \
scanner_zip="sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux-aarch64.zip"; \
base_url="https://binaries.sonarsource.com/Distribution/sonar-scanner-cli"; \
curl -fsSL "${base_url}/${scanner_zip}" -o "/tmp/${scanner_zip}"; \
curl -fsSL "${base_url}/${scanner_zip}.sha256" -o "/tmp/${scanner_zip}.sha256"; \
printf '%s %s\n' "$(cat "/tmp/${scanner_zip}.sha256")" "/tmp/${scanner_zip}" | sha256sum -c -; \
unzip -q "/tmp/${scanner_zip}" -d /opt; \
ln -s "/opt/sonar-scanner-${SONAR_SCANNER_VERSION}-linux-aarch64/bin/sonar-scanner" /usr/local/bin/sonar-scanner; \
rm -f "/tmp/${scanner_zip}" "/tmp/${scanner_zip}.sha256"
RUN set -eux; \
trivy_tgz="trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz"; \
curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_tgz}" -o "/tmp/${trivy_tgz}"; \
tar -C /usr/local/bin -xzf "/tmp/${trivy_tgz}" trivy; \
rm -f "/tmp/${trivy_tgz}"; \
trivy --version; \
sonar-scanner -v
RUN set -eux; \
mkdir -p "${TRIVY_CACHE_DIR}"; \
trivy image --download-db-only --cache-dir "${TRIVY_CACHE_DIR}"; \
chmod -R a+rX "${TRIVY_CACHE_DIR}"; \
mkdir -p /workspace; \
chown quality-tools:quality-tools /workspace
WORKDIR /workspace
USER quality-tools

View File

@ -1,136 +0,0 @@
"""Small credential-free Python execution service for Hermes chat tenants."""
from __future__ import annotations
import json
import os
import resource
import signal
import subprocess
import tempfile
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
HOST = "0.0.0.0"
PORT = 9080
MAX_REQUEST_BYTES = 160 * 1024
MAX_CODE_BYTES = 128 * 1024
MAX_OUTPUT_BYTES = 100 * 1024
WORKSPACE = Path("/workspace")
def _child_limits() -> None:
"""Apply conservative CPU, memory, process, file, and descriptor limits."""
os.setsid()
resource.setrlimit(resource.RLIMIT_CPU, (35, 35))
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024,) * 2)
resource.setrlimit(resource.RLIMIT_NPROC, (32, 32))
resource.setrlimit(resource.RLIMIT_FSIZE, (32 * 1024 * 1024,) * 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
def _execute(code: str) -> dict[str, object]:
"""Run one isolated Python subprocess and return bounded output."""
with tempfile.NamedTemporaryFile(
mode="w", suffix=".py", prefix="hermes-", dir="/tmp", delete=False
) as script:
script.write(code)
script_path = script.name
env = {
"HOME": str(WORKSPACE),
"LANG": "C.UTF-8",
"LC_ALL": "C.UTF-8",
"PATH": "/usr/local/bin:/usr/bin:/bin",
"PYTHONDONTWRITEBYTECODE": "1",
}
try:
process = subprocess.Popen(
["python", "-I", "-B", script_path],
cwd=WORKSPACE,
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=_child_limits,
)
try:
stdout, stderr = process.communicate(timeout=45)
timed_out = False
except subprocess.TimeoutExpired:
timed_out = True
os.killpg(process.pid, signal.SIGKILL)
stdout, stderr = process.communicate()
return {
"success": process.returncode == 0 and not timed_out,
"exit_code": process.returncode,
"timed_out": timed_out,
"stdout": stdout[:MAX_OUTPUT_BYTES].decode("utf-8", errors="replace"),
"stderr": stderr[:MAX_OUTPUT_BYTES].decode("utf-8", errors="replace"),
"output_truncated": (
len(stdout) > MAX_OUTPUT_BYTES or len(stderr) > MAX_OUTPUT_BYTES
),
}
finally:
try:
os.unlink(script_path)
except FileNotFoundError:
pass
class Handler(BaseHTTPRequestHandler):
"""Serve health and bounded Python execution requests."""
server_version = "HermesChatSandbox/1"
def log_message(self, format_string: str, *args: object) -> None:
"""Keep normal request logs concise and free of request bodies."""
print(f"sandbox: {self.address_string()} {format_string % args}", flush=True)
def _json(self, status: int, payload: dict[str, object]) -> None:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Content-Type-Options", "nosniff")
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None:
"""Return a minimal unauthenticated health response."""
if self.path == "/health":
self._json(200, {"status": "ok"})
else:
self._json(404, {"error": "not found"})
def do_POST(self) -> None:
"""Validate and execute a Python request from the matching tenant pod."""
if self.path != "/v1/execute":
self._json(404, {"error": "not found"})
return
try:
length = int(self.headers.get("Content-Length", "0"))
except ValueError:
self._json(400, {"error": "invalid content length"})
return
if length <= 0 or length > MAX_REQUEST_BYTES:
self._json(413, {"error": "request too large"})
return
try:
payload = json.loads(self.rfile.read(length))
except (json.JSONDecodeError, UnicodeDecodeError):
self._json(400, {"error": "invalid JSON"})
return
code = payload.get("code") if isinstance(payload, dict) else None
if not isinstance(code, str) or not code.strip():
self._json(400, {"error": "code is required"})
return
if len(code.encode("utf-8")) > MAX_CODE_BYTES:
self._json(413, {"error": "code exceeds 128 KiB"})
return
self._json(200, _execute(code))
if __name__ == "__main__":
WORKSPACE.mkdir(parents=True, exist_ok=True)
ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()

View File

@ -1,8 +0,0 @@
"""Bundled public-page extraction provider for isolated Hermes chat."""
from plugins.web.public_extract.provider import PublicExtractProvider
def register(ctx) -> None:
"""Register the credential-free extraction provider."""
ctx.register_web_search_provider(PublicExtractProvider())

View File

@ -1,7 +0,0 @@
name: web-public-extract
version: 1.0.0
description: Credential-free extraction for public HTML and text pages.
author: bstein.dev
kind: backend
provides_web_providers:
- public-extract

View File

@ -1,142 +0,0 @@
"""Credential-free extraction of bounded public HTML and text pages."""
from __future__ import annotations
from html.parser import HTMLParser
from typing import Any
from urllib.parse import urljoin
import httpx
from agent.web_search_provider import WebSearchProvider
from tools.url_safety import is_safe_url
from tools.website_policy import check_website_access
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
MAX_REDIRECTS = 5
class _VisibleTextParser(HTMLParser):
"""Collect readable text while discarding scripts, styles, and chrome."""
_ignored = {"script", "style", "noscript", "svg", "nav", "footer"}
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._ignore_depth = 0
self._title_depth = 0
self.title: list[str] = []
self.text: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
if tag in self._ignored:
self._ignore_depth += 1
if tag == "title":
self._title_depth += 1
if not self._ignore_depth and tag in {"p", "div", "section", "article", "li", "br", "h1", "h2", "h3", "h4"}:
self.text.append("\n")
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
if tag == "title" and self._title_depth:
self._title_depth -= 1
if tag in self._ignored and self._ignore_depth:
self._ignore_depth -= 1
if not self._ignore_depth and tag in {"p", "div", "section", "article", "li", "h1", "h2", "h3", "h4"}:
self.text.append("\n")
def handle_data(self, data: str) -> None:
value = " ".join(data.split())
if not value:
return
if self._title_depth:
self.title.append(value)
if not self._ignore_depth:
self.text.append(value)
def readable_text(self) -> str:
"""Return normalized paragraphs from collected visible text."""
lines = [" ".join(line.split()) for line in " ".join(self.text).splitlines()]
return "\n\n".join(line for line in lines if line)
def _fetch_public(url: str) -> tuple[str, str, str]:
"""Fetch one public URL with redirect, size, MIME, and policy checks."""
current = url
headers = {
"User-Agent": "HermesPrivateChat/1.0 (+https://chat.hermes.bstein.dev)",
"Accept": "text/html, text/plain;q=0.9, application/xhtml+xml;q=0.8",
}
with httpx.Client(follow_redirects=False, timeout=15.0, headers=headers) as client:
for _ in range(MAX_REDIRECTS + 1):
if not is_safe_url(current):
raise ValueError("URL targets a private or internal network address")
blocked = check_website_access(current)
if blocked:
raise ValueError(blocked.get("message", "URL is blocked by website policy"))
response = client.get(current)
if response.status_code in {301, 302, 303, 307, 308}:
location = response.headers.get("location")
if not location:
raise ValueError("redirect response omitted Location")
current = urljoin(current, location)
continue
response.raise_for_status()
content_type = response.headers.get("content-type", "").lower()
if not any(kind in content_type for kind in ("text/html", "text/plain", "application/xhtml+xml")):
raise ValueError(f"unsupported content type: {content_type or 'unknown'}")
raw = response.content
if len(raw) > MAX_RESPONSE_BYTES:
raise ValueError("page exceeds the 2 MiB extraction limit")
return current, content_type, response.text
raise ValueError("too many redirects")
class PublicExtractProvider(WebSearchProvider):
"""Extract bounded content directly from public pages without credentials."""
@property
def name(self) -> str:
return "public-extract"
@property
def display_name(self) -> str:
return "Public page extractor"
def is_available(self) -> bool:
return True
def supports_search(self) -> bool:
return False
def supports_extract(self) -> bool:
return True
def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
for url in urls[:20]:
try:
final_url, content_type, body = _fetch_public(url)
if "html" in content_type:
parser = _VisibleTextParser()
parser.feed(body)
title = " ".join(parser.title).strip()
content = parser.readable_text()
else:
title = ""
content = body
results.append(
{
"url": final_url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"source": "public-extract"},
}
)
except Exception as exc:
results.append(
{"url": url, "title": "", "content": "", "error": str(exc)}
)
return results

View File

@ -1,75 +0,0 @@
"""Hermes tool for running Python in a separate, credential-free pod."""
from __future__ import annotations
import json
import os
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from tools.registry import registry, tool_error
def _sandbox_url() -> str:
"""Return the tenant-specific sandbox endpoint injected at startup."""
return os.environ.get("HERMES_CODE_SANDBOX_URL", "").strip()
def _sandbox_available() -> bool:
"""Expose the tool only when this tenant has an isolated endpoint."""
return _sandbox_url().startswith("http://hermes-chat-sandbox-")
def execute_python_sandbox(code: str) -> str:
"""Execute Python remotely and return the sandbox's structured result."""
if not isinstance(code, str) or not code.strip():
return tool_error("Python code is required.")
if len(code.encode("utf-8")) > 128 * 1024:
return tool_error("Python code exceeds the 128 KiB request limit.")
request = Request(
_sandbox_url(),
data=json.dumps({"code": code}).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urlopen(request, timeout=55) as response:
payload = response.read(256 * 1024)
return payload.decode("utf-8", errors="replace")
except HTTPError as exc:
detail = exc.read(4096).decode("utf-8", errors="replace")
return tool_error(f"Python sandbox rejected the request: {detail or exc.code}")
except (URLError, TimeoutError, OSError) as exc:
return tool_error(f"Python sandbox is unavailable: {exc}")
registry.register(
name="python_sandbox",
toolset="python_sandbox",
schema={
"name": "python_sandbox",
"description": (
"Run Python in this user's separate credential-free computation "
"sandbox. Use it for statistics, probability, Monte Carlo simulation, "
"data transforms, and calculations. The sandbox has no Kubernetes, "
"Vault, model-provider credentials, or access to other users. Public "
"research belongs in web_search; pass only the data needed for the "
"calculation and print the result."
),
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "A self-contained Python 3 program that prints its result.",
}
},
"required": ["code"],
},
},
handler=lambda args, **_: execute_python_sandbox(args.get("code", "")),
check_fn=_sandbox_available,
emoji="🧮",
max_result_size_chars=200_000,
)

View File

@ -1,101 +0,0 @@
#!/usr/bin/env python3
"""Copy selected legacy Hermes sessions into an isolated user home."""
from __future__ import annotations
import os
from pathlib import Path
import shutil
import sqlite3
from hermes_state import SessionDB
def _copy_if_missing(source: Path, target: Path) -> None:
"""Copy one credential/config file without overwriting user state."""
if source.is_file() and not target.exists():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
def _copy_session(
source: sqlite3.Connection,
target: sqlite3.Connection,
session_id: str,
user_id: str,
) -> bool:
"""Copy a session and its messages, preserving original timestamps."""
source.row_factory = sqlite3.Row
session = source.execute(
"SELECT * FROM sessions WHERE id = ?", (session_id,)
).fetchone()
if session is None:
return False
columns = list(session.keys())
values = [user_id if column == "user_id" else session[column] for column in columns]
placeholders = ", ".join("?" for _ in columns)
target.execute(
f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
f"VALUES ({placeholders})",
values,
)
message_columns = [
row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
]
selected_columns = [
column
for column in message_columns
if column in {
row[1]
for row in source.execute("PRAGMA table_info(messages)").fetchall()
}
]
column_sql = ", ".join(selected_columns)
target.execute(
f"INSERT OR IGNORE INTO messages ({column_sql}) "
f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
(session_id,),
)
return True
def main() -> None:
"""Create the isolated home and perform the idempotent session copy."""
source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
target_home = Path(os.environ["HERMES_HOME"])
user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
session_ids = [
value.strip()
for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
if value.strip()
]
target_home.mkdir(parents=True, exist_ok=True)
(target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
(target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
(target_home / "logs").mkdir(parents=True, exist_ok=True)
for filename in (".env", "auth.json"):
_copy_if_missing(source_home / filename, target_home / filename)
source_db = source_home / "state.db"
target_db = target_home / "state.db"
SessionDB(db_path=target_db).close()
copied = 0
if source_db.is_file() and session_ids:
source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
target = sqlite3.connect(target_db)
target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
try:
with target:
for session_id in session_ids:
copied += int(_copy_session(source, target, session_id, user_id))
finally:
target.close()
source.close()
print(f"isolated Hermes home ready; migrated_sessions={copied}")
if __name__ == "__main__":
main()

View File

@ -12,7 +12,7 @@ all:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
ansible_user: debian ansible_user: debian
roleset: minipc_gpu roleset: minipc_gpu
dedicated_hosts: baremetal:
hosts: hosts:
titan-db: titan-db:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
@ -22,3 +22,7 @@ all:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
ansible_user: jump ansible_user: jump
roleset: jumphost roleset: jumphost
oceanus:
ansible_host: REPLACE_ME
ansible_user: validator
roleset: validator

View File

@ -14,6 +14,13 @@
- common - common
- titan_jh - titan_jh
- name: Configure oceanus validator host
hosts: oceanus
gather_facts: true
roles:
- common
- oceanus_base
- name: Prepare hybrid tethys node - name: Prepare hybrid tethys node
hosts: titan-24 hosts: titan-24
gather_facts: true gather_facts: true

View File

@ -0,0 +1,6 @@
# hosts/roles/oceanus_base/tasks/main.yaml
---
- name: Placeholder for oceanus base configuration
ansible.builtin.debug:
msg: "Install validator prerequisites and monitoring exporters here."
tags: ['oceanus']

Some files were not shown because too many files have changed in this diff Show More