Compare commits
9 Commits
wt/t_f1593
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4743b678fc | ||
|
|
e9f24b1053 | ||
|
|
8b6195c2e6 | ||
|
|
831d6f817c | ||
|
|
6afc1260dc | ||
|
|
596b2df8e6 | ||
|
|
3c0c868708 | ||
|
|
f167d7d17d | ||
|
|
7018d4ca41 |
@ -1,6 +0,0 @@
|
||||
misconfigurations:
|
||||
- id: KSV-0041
|
||||
paths:
|
||||
- deploy/clusterrole.yaml
|
||||
expired_at: 2026-05-22
|
||||
statement: Soteria copies restic credentials into target namespaces for backup Jobs; replace with a narrower per-namespace secret distribution model.
|
||||
@ -21,7 +21,7 @@ ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 \
|
||||
GOOS=${TARGETOS:-linux} \
|
||||
GOARCH=${TARGETARCH:-$(go env GOARCH)} \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/soteria ./cmd/soteria
|
||||
go build -buildvcs=false -trimpath -ldflags="-s -w" -o /out/soteria ./cmd/soteria
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
COPY --from=builder /out/soteria /soteria
|
||||
|
||||
6
Dockerfile.runtime
Normal file
6
Dockerfile.runtime
Normal file
@ -0,0 +1,6 @@
|
||||
# Jenkins compiles on its scratch PVC; image assembly needs no toolchain layers.
|
||||
FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
|
||||
COPY --chown=65532:65532 soteria /soteria
|
||||
USER 65532:65532
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/soteria"]
|
||||
184
Jenkinsfile
vendored
184
Jenkinsfile
vendored
@ -2,10 +2,15 @@ pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'tester'
|
||||
// Build scratch belongs on storage volumes, not the node runtime USB drive.
|
||||
workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
@ -14,10 +19,9 @@ spec:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-06
|
||||
- {key: hardware, operator: In, values: [rpi5, rpi4]}
|
||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-05, titan-06, titan-08, titan-11, titan-12, titan-13, titan-14, titan-15, titan-17, titan-18, titan-19]}
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
@ -38,27 +42,40 @@ spec:
|
||||
jenkins/jenkins-jenkins-agent: "true"
|
||||
containers:
|
||||
- name: builder
|
||||
image: registry.bstein.dev/bstein/docker:27
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command:
|
||||
- cat
|
||||
- /busybox/cat
|
||||
tty: true
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://localhost:2375
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 2Gi
|
||||
volumeMounts:
|
||||
- name: workspace-volume
|
||||
mountPath: /home/jenkins/agent
|
||||
- name: docker-config-writable
|
||||
mountPath: /root/.docker
|
||||
- name: harbor-config
|
||||
mountPath: /docker-config
|
||||
mountPath: /kaniko/.docker
|
||||
- name: tester
|
||||
image: registry.bstein.dev/bstein/golang:1.25-bookworm
|
||||
command:
|
||||
- cat
|
||||
tty: true
|
||||
resources:
|
||||
requests: {cpu: 200m, memory: 512Mi}
|
||||
limits: {cpu: 1500m, memory: 2Gi}
|
||||
volumeMounts:
|
||||
- name: workspace-volume
|
||||
mountPath: /home/jenkins/agent
|
||||
- name: ui-builder
|
||||
image: node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402
|
||||
command: [cat]
|
||||
tty: true
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 64Mi}
|
||||
limits: {cpu: 1000m, memory: 512Mi}
|
||||
volumeMounts:
|
||||
- name: workspace-volume
|
||||
mountPath: /home/jenkins/agent
|
||||
@ -73,18 +90,17 @@ spec:
|
||||
volumes:
|
||||
- name: docker-config-writable
|
||||
emptyDir: {}
|
||||
- name: harbor-config
|
||||
secret:
|
||||
secretName: harbor-robot-pipeline
|
||||
items:
|
||||
- key: .dockerconfigjson
|
||||
path: config.json
|
||||
- name: workspace-volume
|
||||
emptyDir: {}
|
||||
"""
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
|
||||
NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
|
||||
SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
|
||||
TMPDIR = '/home/jenkins/agent/.cache/tmp'
|
||||
GOCACHE = '/home/jenkins/agent/.cache/go-build'
|
||||
GOMODCACHE = '/home/jenkins/agent/.cache/go-mod'
|
||||
GOTMPDIR = '/home/jenkins/agent/.cache/go-tmp'
|
||||
SUITE_NAME = 'soteria'
|
||||
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
|
||||
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
|
||||
@ -111,6 +127,11 @@ spec:
|
||||
pollSCM('H/5 * * * *')
|
||||
}
|
||||
stages {
|
||||
stage('Prepare build scratch') {
|
||||
steps {
|
||||
sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
|
||||
}
|
||||
}
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout scm
|
||||
@ -121,12 +142,56 @@ spec:
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p "$GOCACHE" "$GOMODCACHE" "$GOTMPDIR"
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends bash git jq curl python3 ripgrep >/dev/null
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Run quality gate') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends jq python3 ripgrep >/dev/null
|
||||
mkdir -p build
|
||||
set +e
|
||||
bash scripts/check.sh
|
||||
gate_rc=$?
|
||||
set -e
|
||||
if [ ! -f build/go-test.json ]; then
|
||||
: > build/go-test.json
|
||||
fi
|
||||
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_passed=$((tests_total - tests_failed - tests_skipped))
|
||||
if [ "${tests_passed}" -lt 0 ]; then
|
||||
tests_passed=0
|
||||
fi
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
source_files_total="$(jq -r '.source_files_total // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
cat > build/test-summary.json <<EOF
|
||||
{
|
||||
"tests": ${tests_total},
|
||||
"passed": ${tests_passed},
|
||||
"failed": ${tests_failed},
|
||||
"errors": ${tests_errors},
|
||||
"skipped": ${tests_skipped},
|
||||
"coverage_percent": ${coverage_percent},
|
||||
"source_files_total": ${source_files_total},
|
||||
"source_lines_over_500": ${over_500}
|
||||
}
|
||||
EOF
|
||||
printf '%s\n' "${gate_rc}" > build/test.exitcode
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Collect SonarQube and supply-chain evidence') {
|
||||
steps {
|
||||
container('quality-tools') {
|
||||
@ -138,6 +203,8 @@ spec:
|
||||
"-Dsonar.login=${SONARQUBE_TOKEN}"
|
||||
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
|
||||
"-Dsonar.qualitygate.wait=true"
|
||||
"-Dsonar.qualitygate.timeout=300"
|
||||
"-Dsonar.sources=."
|
||||
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**"
|
||||
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
|
||||
@ -174,14 +241,11 @@ EOF
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Run quality gate') {
|
||||
stage('Read quality evidence') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends jq python3 ripgrep >/dev/null
|
||||
mkdir -p build
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
@ -227,37 +291,7 @@ if not ironbank_report.exists():
|
||||
ironbank_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
ironbank_report.write_text(json.dumps(ironbank_payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
||||
PY
|
||||
set +e
|
||||
bash scripts/check.sh
|
||||
gate_rc=$?
|
||||
set -e
|
||||
if [ ! -f build/go-test.json ]; then
|
||||
: > build/go-test.json
|
||||
fi
|
||||
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_passed=$((tests_total - tests_failed - tests_skipped))
|
||||
if [ "${tests_passed}" -lt 0 ]; then
|
||||
tests_passed=0
|
||||
fi
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
source_files_total="$(jq -r '.source_files_total // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
cat > build/test-summary.json <<EOF
|
||||
{
|
||||
"tests": ${tests_total},
|
||||
"passed": ${tests_passed},
|
||||
"failed": ${tests_failed},
|
||||
"errors": ${tests_errors},
|
||||
"skipped": ${tests_skipped},
|
||||
"coverage_percent": ${coverage_percent},
|
||||
"source_files_total": ${source_files_total},
|
||||
"source_lines_over_500": ${over_500}
|
||||
}
|
||||
EOF
|
||||
printf '%s\n' "${gate_rc}" > build/test.exitcode
|
||||
|
||||
'''
|
||||
}
|
||||
}
|
||||
@ -511,7 +545,7 @@ PY
|
||||
expression { return params.PUBLISH_IMAGES }
|
||||
}
|
||||
steps {
|
||||
container('builder') {
|
||||
container('tester') {
|
||||
script {
|
||||
sh 'git config --global --add safe.directory /home/jenkins/agent/workspace/Soteria'
|
||||
def semver = sh(returnStdout: true, script: 'git describe --tags --exact-match || true').trim()
|
||||
@ -524,20 +558,23 @@ PY
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Buildx setup') {
|
||||
stage('Build release on scratch') {
|
||||
when {
|
||||
expression { return params.PUBLISH_IMAGES }
|
||||
}
|
||||
steps {
|
||||
container('builder') {
|
||||
container('ui-builder') {
|
||||
sh 'cd web && npm ci && npm run build'
|
||||
}
|
||||
container('tester') {
|
||||
// Reuse the tested Go cache and put compiler writes on the workspace PVC.
|
||||
sh '''
|
||||
set -eu
|
||||
seq 1 10 | while read _; do
|
||||
docker info && break || sleep 2
|
||||
done
|
||||
BUILDER_NAME="soteria-${BUILD_NUMBER}"
|
||||
docker buildx rm "${BUILDER_NAME}" >/dev/null 2>&1 || true
|
||||
docker buildx create --name "${BUILDER_NAME}" --driver docker-container --driver-opt image=registry.bstein.dev/bstein/buildkit:buildx-stable-1 --bootstrap --use
|
||||
mkdir -p build/image internal/server/ui-dist
|
||||
cp -R web/dist/. internal/server/ui-dist/
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -p 2 -buildvcs=false -trimpath -ldflags="-s -w" -o build/image/soteria ./cmd/soteria
|
||||
chmod 0755 build/image/soteria
|
||||
cp Dockerfile.runtime build/image/Dockerfile
|
||||
'''
|
||||
}
|
||||
}
|
||||
@ -551,13 +588,16 @@ PY
|
||||
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
|
||||
sh '''
|
||||
set -eu
|
||||
set +x
|
||||
VERSION_TAG=$(cut -d= -f2 build.env)
|
||||
printf '%s' "${HARBOR_PASSWORD}" | docker login registry.bstein.dev -u "${HARBOR_USERNAME}" --password-stdin
|
||||
docker buildx build --platform linux/arm64 \
|
||||
--provenance=false \
|
||||
--tag registry.bstein.dev/bstein/soteria:${VERSION_TAG} \
|
||||
--tag registry.bstein.dev/bstein/soteria:latest \
|
||||
--push .
|
||||
auth=$(printf '%s:%s' "${HARBOR_USERNAME}" "${HARBOR_PASSWORD}" | base64 | tr -d '\\n')
|
||||
(umask 077; printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\\n' "${auth}" > /kaniko/.docker/config.json)
|
||||
trap 'rm -f /kaniko/.docker/config.json' EXIT
|
||||
/kaniko/executor \
|
||||
--context "${WORKSPACE}/build/image" \
|
||||
--dockerfile "${WORKSPACE}/build/image/Dockerfile" \
|
||||
--destination "registry.bstein.dev/bstein/soteria:${VERSION_TAG}" \
|
||||
--destination registry.bstein.dev/bstein/soteria:latest
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
28
deploy/NOTES.md
Normal file
28
deploy/NOTES.md
Normal file
@ -0,0 +1,28 @@
|
||||
# Default deployment permissions
|
||||
|
||||
This deployment uses the Longhorn backend. It reads inventory across namespaces
|
||||
and updates only `soteria-policies` and `soteria-backup-usage` in its own namespace.
|
||||
The empty Secret declarations create those records on first deployment; they
|
||||
deliberately contain no data field, so application-written state is preserved.
|
||||
The optional `soteria-restic` credential is readable by exact name only.
|
||||
|
||||
Changing the state or credential names also requires updating the Role. Do not
|
||||
restore cluster-wide Secret access to accommodate another deployment.
|
||||
|
||||
The optional restic backend needs additional, explicitly approved Roles and
|
||||
RoleBindings in each backup/restore target namespace: Job creation and the
|
||||
credential-copy operations in `internal/k8s/jobs.go`. Those grants are not part
|
||||
of this Longhorn deployment. Review credential distribution before enabling
|
||||
restic; Kubernetes RBAC cannot restrict Secret creation by resource name.
|
||||
|
||||
## CI image construction
|
||||
|
||||
Jenkins compiles the UI and ARM64 Go binary in its `ci-scratch` workspace PVC.
|
||||
Compiler and package caches also use that PVC. The final Kaniko step receives
|
||||
only `build/image/` and `Dockerfile.runtime`, so it packages the binary without
|
||||
unpacking Node/Go toolchains or compiling on the node's runtime filesystem.
|
||||
The ordinary multistage `Dockerfile` remains available for workstation builds.
|
||||
|
||||
Keep `CGO_ENABLED=0`, the UI embedding step and the runtime architecture aligned.
|
||||
The runtime image stays nonroot and exposes the same port and entrypoint. Image
|
||||
publication still requires all existing quality and supply-chain gates.
|
||||
@ -8,9 +8,6 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims", "persistentvolumes"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list", "create", "update", "delete"]
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["get", "list", "create"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
@ -7,5 +7,6 @@ resources:
|
||||
- serviceaccount.yaml
|
||||
- clusterrole.yaml
|
||||
- clusterrolebinding.yaml
|
||||
- state-access.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
39
deploy/state-access.yaml
Normal file
39
deploy/state-access.yaml
Normal file
@ -0,0 +1,39 @@
|
||||
# The default Longhorn mode writes only its own policy and usage records.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: soteria-policies
|
||||
type: Opaque
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: soteria-backup-usage
|
||||
type: Opaque
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: soteria-state
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["soteria-policies", "soteria-backup-usage"]
|
||||
verbs: ["get", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["soteria-restic"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: soteria-state
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: soteria-state
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: soteria
|
||||
namespace: soteria
|
||||
@ -206,10 +206,19 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
|
||||
|
||||
switch s.cfg.BackupDriver {
|
||||
case "longhorn":
|
||||
volumeName, _, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||
volumeName, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||
if err != nil {
|
||||
return api.BackupResponse{}, "validation_error", err
|
||||
}
|
||||
storageClass := ""
|
||||
if pvc != nil && pvc.Spec.StorageClassName != nil {
|
||||
storageClass = *pvc.Spec.StorageClassName
|
||||
}
|
||||
// Apply data-location exclusions to manual and namespace requests as
|
||||
// well as scheduled backups, before invoking the Longhorn backend.
|
||||
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
|
||||
return api.BackupResponse{}, "validation_error", errors.New(reason)
|
||||
}
|
||||
|
||||
backupID := backupName("backup", req.Namespace+"-"+req.PVC)
|
||||
response := api.BackupResponse{
|
||||
|
||||
54
internal/server/longhorn_live_policy_test.go
Normal file
54
internal/server/longhorn_live_policy_test.go
Normal file
@ -0,0 +1,54 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"scm.bstein.dev/bstein/soteria/internal/api"
|
||||
"scm.bstein.dev/bstein/soteria/internal/config"
|
||||
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
||||
)
|
||||
|
||||
// TestLiveRWOLonghornPolicy protects live workloads without a second mount.
|
||||
func TestLiveRWOLonghornPolicy(t *testing.T) {
|
||||
client := &policyCycleTestKubeClient{
|
||||
inventoryTestKubeClient: &inventoryTestKubeClient{
|
||||
fakeKubeClient: &fakeKubeClient{
|
||||
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
|
||||
pvcMounts: map[string][]k8s.PVCMount{"apps/data": {{PodName: "database-0", NodeName: "worker", Phase: "Running"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
backend := &fakeLonghornClient{}
|
||||
srv := &Server{
|
||||
cfg: &config.Config{BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1},
|
||||
client: client, longhorn: backend, metrics: newTelemetry(),
|
||||
policies: map[string]api.BackupPolicy{"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}},
|
||||
}
|
||||
srv.runPolicyCycle(context.Background())
|
||||
if backend.createSnapshotName == "" {
|
||||
t.Fatal("live RWO workload did not receive a Longhorn snapshot")
|
||||
}
|
||||
if len(client.backupRequests) != 0 {
|
||||
t.Fatal("Longhorn policy attempted to launch a filesystem-mount job")
|
||||
}
|
||||
if metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}) != 1 {
|
||||
t.Fatal("expected one successful policy backup")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLonghornExclusionAppliesToManualRequests prevents bypassing local-only policy.
|
||||
func TestLonghornExclusionAppliesToManualRequests(t *testing.T) {
|
||||
backend := &fakeLonghornClient{}
|
||||
srv := &Server{
|
||||
cfg: &config.Config{BackupDriver: "longhorn", ExcludedPVCs: []string{"hermes/*"}},
|
||||
client: &fakeKubeClient{}, longhorn: backend,
|
||||
}
|
||||
for _, dryRun := range []bool{false, true} {
|
||||
_, code, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "hermes", PVC: "workspace", DryRun: dryRun}, "test")
|
||||
if err == nil || code != "validation_error" || backend.createSnapshotName != "" {
|
||||
t.Fatal("excluded Longhorn PVC reached the backup backend")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -162,7 +162,11 @@ func (t *telemetry) RecordInventory(inv api.InventoryResponse) {
|
||||
}
|
||||
reasonLabels["reason"] = reason
|
||||
setMetric(t.pvcBackupHealthReason, reasonLabels, 1)
|
||||
setMetric(t.pvcBackupHealth, labels, boolGauge(pvc.Healthy))
|
||||
if pvc.Healthy {
|
||||
setMetric(t.pvcBackupHealth, labels, 1)
|
||||
} else {
|
||||
setMetric(t.pvcBackupHealth, labels, 0)
|
||||
}
|
||||
if pvc.LastBackupAt == "" {
|
||||
continue
|
||||
}
|
||||
@ -321,15 +325,6 @@ func incMetric(target map[string]metricSample, labels map[string]string) {
|
||||
target[key] = sample
|
||||
}
|
||||
|
||||
// boolGauge converts a boolean state into the Prometheus gauge convention of
|
||||
// 1 for true and 0 for false.
|
||||
func boolGauge(value bool) float64 {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func setMetric(target map[string]metricSample, labels map[string]string, value float64) {
|
||||
key := metricKey(labels)
|
||||
target[key] = metricSample{labels: cloneLabels(labels), value: value}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -161,55 +160,6 @@ func TestTelemetryRecordInventoryPopulatesAndResetsMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoolGaugeMapsHealthToPrometheusValues(t *testing.T) {
|
||||
if got := boolGauge(true); got != 1 {
|
||||
t.Fatalf("expected boolGauge(true) to be 1, got %v", got)
|
||||
}
|
||||
if got := boolGauge(false); got != 0 {
|
||||
t.Fatalf("expected boolGauge(false) to be 0, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryRecordInventoryHealthGaugeIdentity(t *testing.T) {
|
||||
telemetry := newTelemetry()
|
||||
|
||||
telemetry.RecordInventory(api.InventoryResponse{
|
||||
Namespaces: []api.NamespaceInventory{
|
||||
{
|
||||
Name: "apps",
|
||||
PVCs: []api.PVCInventory{
|
||||
{
|
||||
Namespace: "apps",
|
||||
PVC: "data",
|
||||
Volume: "pv-apps-data",
|
||||
Driver: "restic",
|
||||
Healthy: true,
|
||||
},
|
||||
{
|
||||
Namespace: "apps",
|
||||
PVC: "cache",
|
||||
Volume: "pv-apps-cache",
|
||||
Driver: "longhorn",
|
||||
Healthy: false,
|
||||
HealthReason: "stale",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
rendered := telemetry.render()
|
||||
for _, line := range []string{
|
||||
"# TYPE pvc_backup_health gauge",
|
||||
`pvc_backup_health{driver="restic",namespace="apps",pvc="data",volume="pv-apps-data"} 1`,
|
||||
`pvc_backup_health{driver="longhorn",namespace="apps",pvc="cache",volume="pv-apps-cache"} 0`,
|
||||
} {
|
||||
if !strings.Contains(rendered, line+"\n") {
|
||||
t.Fatalf("expected rendered metrics to contain %q, got:\n%s", line, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryRecordB2UsageTracksBucketsAndFallbackTimestamp(t *testing.T) {
|
||||
telemetry := newTelemetry()
|
||||
scannedAt := time.Date(2026, 4, 20, 16, 30, 0, 0, time.UTC)
|
||||
|
||||
@ -103,15 +103,19 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
||||
s.metrics.RecordPolicyBackup("excluded")
|
||||
continue
|
||||
}
|
||||
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||
if err != nil {
|
||||
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||
continue
|
||||
}
|
||||
if blocked {
|
||||
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||
continue
|
||||
// Only restic needs a second filesystem mount. Longhorn snapshots use
|
||||
// the existing engine and must protect live RWO workloads too.
|
||||
if s.cfg.BackupDriver == "restic" {
|
||||
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||
if err != nil {
|
||||
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||
continue
|
||||
}
|
||||
if blocked {
|
||||
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Never enqueue a new policy backup while one is already active for this PVC.
|
||||
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user