Compare commits

..

9 Commits

13 changed files with 270 additions and 153 deletions

View File

@ -1,6 +0,0 @@
misconfigurations:
- id: KSV-0041
paths:
- deploy/clusterrole.yaml
expired_at: 2026-05-22
statement: Soteria copies restic credentials into target namespaces for backup Jobs; replace with a narrower per-namespace secret distribution model.

View File

@ -21,7 +21,7 @@ ARG TARGETARCH
RUN CGO_ENABLED=0 \ RUN CGO_ENABLED=0 \
GOOS=${TARGETOS:-linux} \ GOOS=${TARGETOS:-linux} \
GOARCH=${TARGETARCH:-$(go env GOARCH)} \ GOARCH=${TARGETARCH:-$(go env GOARCH)} \
go build -trimpath -ldflags="-s -w" -o /out/soteria ./cmd/soteria go build -buildvcs=false -trimpath -ldflags="-s -w" -o /out/soteria ./cmd/soteria
FROM gcr.io/distroless/static-debian12:nonroot FROM gcr.io/distroless/static-debian12:nonroot
COPY --from=builder /out/soteria /soteria COPY --from=builder /out/soteria /soteria

6
Dockerfile.runtime Normal file
View File

@ -0,0 +1,6 @@
# Jenkins compiles on its scratch PVC; image assembly needs no toolchain layers.
FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
COPY --chown=65532:65532 soteria /soteria
USER 65532:65532
EXPOSE 8080
ENTRYPOINT ["/soteria"]

184
Jenkinsfile vendored
View File

@ -2,10 +2,15 @@ pipeline {
agent { agent {
kubernetes { kubernetes {
defaultContainer 'tester' defaultContainer 'tester'
// Build scratch belongs on storage volumes, not the node runtime USB drive.
workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
yaml """ yaml """
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
spec: spec:
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector: nodeSelector:
kubernetes.io/arch: arm64 kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true" node-role.kubernetes.io/worker: "true"
@ -14,10 +19,9 @@ spec:
requiredDuringSchedulingIgnoredDuringExecution: requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms: nodeSelectorTerms:
- matchExpressions: - matchExpressions:
- key: kubernetes.io/hostname - {key: hardware, operator: In, values: [rpi5, rpi4]}
operator: NotIn - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
values: - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-05, titan-06, titan-08, titan-11, titan-12, titan-13, titan-14, titan-15, titan-17, titan-18, titan-19]}
- titan-06
preferredDuringSchedulingIgnoredDuringExecution: preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100 - weight: 100
preference: preference:
@ -38,27 +42,40 @@ spec:
jenkins/jenkins-jenkins-agent: "true" jenkins/jenkins-jenkins-agent: "true"
containers: containers:
- name: builder - name: builder
image: registry.bstein.dev/bstein/docker:27 image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: command:
- cat - /busybox/cat
tty: true tty: true
env: resources:
- name: DOCKER_HOST requests:
value: tcp://localhost:2375 cpu: 100m
- name: DOCKER_TLS_CERTDIR memory: 512Mi
value: "" limits:
cpu: 1500m
memory: 2Gi
volumeMounts: volumeMounts:
- name: workspace-volume - name: workspace-volume
mountPath: /home/jenkins/agent mountPath: /home/jenkins/agent
- name: docker-config-writable - name: docker-config-writable
mountPath: /root/.docker mountPath: /kaniko/.docker
- name: harbor-config
mountPath: /docker-config
- name: tester - name: tester
image: registry.bstein.dev/bstein/golang:1.25-bookworm image: registry.bstein.dev/bstein/golang:1.25-bookworm
command: command:
- cat - cat
tty: true tty: true
resources:
requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: 1500m, memory: 2Gi}
volumeMounts:
- name: workspace-volume
mountPath: /home/jenkins/agent
- name: ui-builder
image: node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402
command: [cat]
tty: true
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 1000m, memory: 512Mi}
volumeMounts: volumeMounts:
- name: workspace-volume - name: workspace-volume
mountPath: /home/jenkins/agent mountPath: /home/jenkins/agent
@ -73,18 +90,17 @@ spec:
volumes: volumes:
- name: docker-config-writable - name: docker-config-writable
emptyDir: {} emptyDir: {}
- name: harbor-config
secret:
secretName: harbor-robot-pipeline
items:
- key: .dockerconfigjson
path: config.json
- name: workspace-volume
emptyDir: {}
""" """
} }
} }
environment { environment {
PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
TMPDIR = '/home/jenkins/agent/.cache/tmp'
GOCACHE = '/home/jenkins/agent/.cache/go-build'
GOMODCACHE = '/home/jenkins/agent/.cache/go-mod'
GOTMPDIR = '/home/jenkins/agent/.cache/go-tmp'
SUITE_NAME = 'soteria' SUITE_NAME = 'soteria'
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000' SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
@ -111,6 +127,11 @@ spec:
pollSCM('H/5 * * * *') pollSCM('H/5 * * * *')
} }
stages { stages {
stage('Prepare build scratch') {
steps {
sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
}
}
stage('Checkout') { stage('Checkout') {
steps { steps {
checkout scm checkout scm
@ -121,12 +142,56 @@ spec:
container('tester') { container('tester') {
sh ''' sh '''
set -eu set -eu
mkdir -p "$GOCACHE" "$GOMODCACHE" "$GOTMPDIR"
apt-get update >/dev/null apt-get update >/dev/null
apt-get install -y --no-install-recommends bash git jq curl python3 ripgrep >/dev/null apt-get install -y --no-install-recommends bash git jq curl python3 ripgrep >/dev/null
''' '''
} }
} }
} }
stage('Run quality gate') {
steps {
container('tester') {
sh '''
set -eu
apt-get update >/dev/null
apt-get install -y --no-install-recommends jq python3 ripgrep >/dev/null
mkdir -p build
set +e
bash scripts/check.sh
gate_rc=$?
set -e
if [ ! -f build/go-test.json ]; then
: > build/go-test.json
fi
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_passed=$((tests_total - tests_failed - tests_skipped))
if [ "${tests_passed}" -lt 0 ]; then
tests_passed=0
fi
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
source_files_total="$(jq -r '.source_files_total // 0' build/quality-summary.json 2>/dev/null || echo 0)"
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
cat > build/test-summary.json <<EOF
{
"tests": ${tests_total},
"passed": ${tests_passed},
"failed": ${tests_failed},
"errors": ${tests_errors},
"skipped": ${tests_skipped},
"coverage_percent": ${coverage_percent},
"source_files_total": ${source_files_total},
"source_lines_over_500": ${over_500}
}
EOF
printf '%s\n' "${gate_rc}" > build/test.exitcode
'''
}
}
}
stage('Collect SonarQube and supply-chain evidence') { stage('Collect SonarQube and supply-chain evidence') {
steps { steps {
container('quality-tools') { container('quality-tools') {
@ -138,6 +203,8 @@ spec:
"-Dsonar.login=${SONARQUBE_TOKEN}" "-Dsonar.login=${SONARQUBE_TOKEN}"
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}" "-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}" "-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.qualitygate.wait=true"
"-Dsonar.qualitygate.timeout=300"
"-Dsonar.sources=." "-Dsonar.sources=."
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**" "-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**"
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
@ -174,14 +241,11 @@ EOF
} }
} }
} }
stage('Run quality gate') { stage('Read quality evidence') {
steps { steps {
container('tester') { container('tester') {
sh ''' sh '''
set -eu set -eu
apt-get update >/dev/null
apt-get install -y --no-install-recommends jq python3 ripgrep >/dev/null
mkdir -p build
python3 - <<'PY' python3 - <<'PY'
import base64 import base64
import json import json
@ -227,37 +291,7 @@ if not ironbank_report.exists():
ironbank_report.parent.mkdir(parents=True, exist_ok=True) ironbank_report.parent.mkdir(parents=True, exist_ok=True)
ironbank_report.write_text(json.dumps(ironbank_payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8") ironbank_report.write_text(json.dumps(ironbank_payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
PY PY
set +e
bash scripts/check.sh
gate_rc=$?
set -e
if [ ! -f build/go-test.json ]; then
: > build/go-test.json
fi
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_passed=$((tests_total - tests_failed - tests_skipped))
if [ "${tests_passed}" -lt 0 ]; then
tests_passed=0
fi
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
source_files_total="$(jq -r '.source_files_total // 0' build/quality-summary.json 2>/dev/null || echo 0)"
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
cat > build/test-summary.json <<EOF
{
"tests": ${tests_total},
"passed": ${tests_passed},
"failed": ${tests_failed},
"errors": ${tests_errors},
"skipped": ${tests_skipped},
"coverage_percent": ${coverage_percent},
"source_files_total": ${source_files_total},
"source_lines_over_500": ${over_500}
}
EOF
printf '%s\n' "${gate_rc}" > build/test.exitcode
''' '''
} }
} }
@ -511,7 +545,7 @@ PY
expression { return params.PUBLISH_IMAGES } expression { return params.PUBLISH_IMAGES }
} }
steps { steps {
container('builder') { container('tester') {
script { script {
sh 'git config --global --add safe.directory /home/jenkins/agent/workspace/Soteria' sh 'git config --global --add safe.directory /home/jenkins/agent/workspace/Soteria'
def semver = sh(returnStdout: true, script: 'git describe --tags --exact-match || true').trim() def semver = sh(returnStdout: true, script: 'git describe --tags --exact-match || true').trim()
@ -524,20 +558,23 @@ PY
} }
} }
} }
stage('Buildx setup') { stage('Build release on scratch') {
when { when {
expression { return params.PUBLISH_IMAGES } expression { return params.PUBLISH_IMAGES }
} }
steps { steps {
container('builder') { container('ui-builder') {
sh 'cd web && npm ci && npm run build'
}
container('tester') {
// Reuse the tested Go cache and put compiler writes on the workspace PVC.
sh ''' sh '''
set -eu set -eu
seq 1 10 | while read _; do mkdir -p build/image internal/server/ui-dist
docker info && break || sleep 2 cp -R web/dist/. internal/server/ui-dist/
done CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -p 2 -buildvcs=false -trimpath -ldflags="-s -w" -o build/image/soteria ./cmd/soteria
BUILDER_NAME="soteria-${BUILD_NUMBER}" chmod 0755 build/image/soteria
docker buildx rm "${BUILDER_NAME}" >/dev/null 2>&1 || true cp Dockerfile.runtime build/image/Dockerfile
docker buildx create --name "${BUILDER_NAME}" --driver docker-container --driver-opt image=registry.bstein.dev/bstein/buildkit:buildx-stable-1 --bootstrap --use
''' '''
} }
} }
@ -551,13 +588,16 @@ PY
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) { withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
sh ''' sh '''
set -eu set -eu
set +x
VERSION_TAG=$(cut -d= -f2 build.env) VERSION_TAG=$(cut -d= -f2 build.env)
printf '%s' "${HARBOR_PASSWORD}" | docker login registry.bstein.dev -u "${HARBOR_USERNAME}" --password-stdin auth=$(printf '%s:%s' "${HARBOR_USERNAME}" "${HARBOR_PASSWORD}" | base64 | tr -d '\\n')
docker buildx build --platform linux/arm64 \ (umask 077; printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\\n' "${auth}" > /kaniko/.docker/config.json)
--provenance=false \ trap 'rm -f /kaniko/.docker/config.json' EXIT
--tag registry.bstein.dev/bstein/soteria:${VERSION_TAG} \ /kaniko/executor \
--tag registry.bstein.dev/bstein/soteria:latest \ --context "${WORKSPACE}/build/image" \
--push . --dockerfile "${WORKSPACE}/build/image/Dockerfile" \
--destination "registry.bstein.dev/bstein/soteria:${VERSION_TAG}" \
--destination registry.bstein.dev/bstein/soteria:latest
''' '''
} }
} }

28
deploy/NOTES.md Normal file
View File

@ -0,0 +1,28 @@
# Default deployment permissions
This deployment uses the Longhorn backend. It reads inventory across namespaces
and updates only `soteria-policies` and `soteria-backup-usage` in its own namespace.
The empty Secret declarations create those records on first deployment; they
deliberately contain no data field, so application-written state is preserved.
The optional `soteria-restic` credential is readable by exact name only.
Changing the state or credential names also requires updating the Role. Do not
restore cluster-wide Secret access to accommodate another deployment.
The optional restic backend needs additional, explicitly approved Roles and
RoleBindings in each backup/restore target namespace: Job creation and the
credential-copy operations in `internal/k8s/jobs.go`. Those grants are not part
of this Longhorn deployment. Review credential distribution before enabling
restic; Kubernetes RBAC cannot restrict Secret creation by resource name.
## CI image construction
Jenkins compiles the UI and ARM64 Go binary in its `ci-scratch` workspace PVC.
Compiler and package caches also use that PVC. The final Kaniko step receives
only `build/image/` and `Dockerfile.runtime`, so it packages the binary without
unpacking Node/Go toolchains or compiling on the node's runtime filesystem.
The ordinary multistage `Dockerfile` remains available for workstation builds.
Keep `CGO_ENABLED=0`, the UI embedding step and the runtime architecture aligned.
The runtime image stays nonroot and exposes the same port and entrypoint. Image
publication still requires all existing quality and supply-chain gates.

View File

@ -8,9 +8,6 @@ rules:
- apiGroups: [""] - apiGroups: [""]
resources: ["persistentvolumeclaims", "persistentvolumes"] resources: ["persistentvolumeclaims", "persistentvolumes"]
verbs: ["get", "list"] verbs: ["get", "list"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "create", "update", "delete"]
- apiGroups: ["batch"] - apiGroups: ["batch"]
resources: ["jobs"] resources: ["jobs"]
verbs: ["get", "list", "create"] verbs: ["get", "list"]

View File

@ -7,5 +7,6 @@ resources:
- serviceaccount.yaml - serviceaccount.yaml
- clusterrole.yaml - clusterrole.yaml
- clusterrolebinding.yaml - clusterrolebinding.yaml
- state-access.yaml
- deployment.yaml - deployment.yaml
- service.yaml - service.yaml

39
deploy/state-access.yaml Normal file
View File

@ -0,0 +1,39 @@
# The default Longhorn mode writes only its own policy and usage records.
apiVersion: v1
kind: Secret
metadata:
name: soteria-policies
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
name: soteria-backup-usage
type: Opaque
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: soteria-state
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["soteria-policies", "soteria-backup-usage"]
verbs: ["get", "update"]
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["soteria-restic"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: soteria-state
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: soteria-state
subjects:
- kind: ServiceAccount
name: soteria
namespace: soteria

View File

@ -206,10 +206,19 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
switch s.cfg.BackupDriver { switch s.cfg.BackupDriver {
case "longhorn": case "longhorn":
volumeName, _, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC) volumeName, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
if err != nil { if err != nil {
return api.BackupResponse{}, "validation_error", err return api.BackupResponse{}, "validation_error", err
} }
storageClass := ""
if pvc != nil && pvc.Spec.StorageClassName != nil {
storageClass = *pvc.Spec.StorageClassName
}
// Apply data-location exclusions to manual and namespace requests as
// well as scheduled backups, before invoking the Longhorn backend.
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
backupID := backupName("backup", req.Namespace+"-"+req.PVC) backupID := backupName("backup", req.Namespace+"-"+req.PVC)
response := api.BackupResponse{ response := api.BackupResponse{

View File

@ -0,0 +1,54 @@
package server
import (
"context"
"testing"
"time"
"scm.bstein.dev/bstein/soteria/internal/api"
"scm.bstein.dev/bstein/soteria/internal/config"
"scm.bstein.dev/bstein/soteria/internal/k8s"
)
// TestLiveRWOLonghornPolicy protects live workloads without a second mount.
func TestLiveRWOLonghornPolicy(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
pvcMounts: map[string][]k8s.PVCMount{"apps/data": {{PodName: "database-0", NodeName: "worker", Phase: "Running"}}},
},
},
}
backend := &fakeLonghornClient{}
srv := &Server{
cfg: &config.Config{BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1},
client: client, longhorn: backend, metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}},
}
srv.runPolicyCycle(context.Background())
if backend.createSnapshotName == "" {
t.Fatal("live RWO workload did not receive a Longhorn snapshot")
}
if len(client.backupRequests) != 0 {
t.Fatal("Longhorn policy attempted to launch a filesystem-mount job")
}
if metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}) != 1 {
t.Fatal("expected one successful policy backup")
}
}
// TestLonghornExclusionAppliesToManualRequests prevents bypassing local-only policy.
func TestLonghornExclusionAppliesToManualRequests(t *testing.T) {
backend := &fakeLonghornClient{}
srv := &Server{
cfg: &config.Config{BackupDriver: "longhorn", ExcludedPVCs: []string{"hermes/*"}},
client: &fakeKubeClient{}, longhorn: backend,
}
for _, dryRun := range []bool{false, true} {
_, code, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "hermes", PVC: "workspace", DryRun: dryRun}, "test")
if err == nil || code != "validation_error" || backend.createSnapshotName != "" {
t.Fatal("excluded Longhorn PVC reached the backup backend")
}
}
}

View File

@ -162,7 +162,11 @@ func (t *telemetry) RecordInventory(inv api.InventoryResponse) {
} }
reasonLabels["reason"] = reason reasonLabels["reason"] = reason
setMetric(t.pvcBackupHealthReason, reasonLabels, 1) setMetric(t.pvcBackupHealthReason, reasonLabels, 1)
setMetric(t.pvcBackupHealth, labels, boolGauge(pvc.Healthy)) if pvc.Healthy {
setMetric(t.pvcBackupHealth, labels, 1)
} else {
setMetric(t.pvcBackupHealth, labels, 0)
}
if pvc.LastBackupAt == "" { if pvc.LastBackupAt == "" {
continue continue
} }
@ -321,15 +325,6 @@ func incMetric(target map[string]metricSample, labels map[string]string) {
target[key] = sample target[key] = sample
} }
// boolGauge converts a boolean state into the Prometheus gauge convention of
// 1 for true and 0 for false.
func boolGauge(value bool) float64 {
if value {
return 1
}
return 0
}
func setMetric(target map[string]metricSample, labels map[string]string, value float64) { func setMetric(target map[string]metricSample, labels map[string]string, value float64) {
key := metricKey(labels) key := metricKey(labels)
target[key] = metricSample{labels: cloneLabels(labels), value: value} target[key] = metricSample{labels: cloneLabels(labels), value: value}

View File

@ -1,7 +1,6 @@
package server package server
import ( import (
"strings"
"testing" "testing"
"time" "time"
@ -161,55 +160,6 @@ func TestTelemetryRecordInventoryPopulatesAndResetsMetrics(t *testing.T) {
} }
} }
func TestBoolGaugeMapsHealthToPrometheusValues(t *testing.T) {
if got := boolGauge(true); got != 1 {
t.Fatalf("expected boolGauge(true) to be 1, got %v", got)
}
if got := boolGauge(false); got != 0 {
t.Fatalf("expected boolGauge(false) to be 0, got %v", got)
}
}
func TestTelemetryRecordInventoryHealthGaugeIdentity(t *testing.T) {
telemetry := newTelemetry()
telemetry.RecordInventory(api.InventoryResponse{
Namespaces: []api.NamespaceInventory{
{
Name: "apps",
PVCs: []api.PVCInventory{
{
Namespace: "apps",
PVC: "data",
Volume: "pv-apps-data",
Driver: "restic",
Healthy: true,
},
{
Namespace: "apps",
PVC: "cache",
Volume: "pv-apps-cache",
Driver: "longhorn",
Healthy: false,
HealthReason: "stale",
},
},
},
},
})
rendered := telemetry.render()
for _, line := range []string{
"# TYPE pvc_backup_health gauge",
`pvc_backup_health{driver="restic",namespace="apps",pvc="data",volume="pv-apps-data"} 1`,
`pvc_backup_health{driver="longhorn",namespace="apps",pvc="cache",volume="pv-apps-cache"} 0`,
} {
if !strings.Contains(rendered, line+"\n") {
t.Fatalf("expected rendered metrics to contain %q, got:\n%s", line, rendered)
}
}
}
func TestTelemetryRecordB2UsageTracksBucketsAndFallbackTimestamp(t *testing.T) { func TestTelemetryRecordB2UsageTracksBucketsAndFallbackTimestamp(t *testing.T) {
telemetry := newTelemetry() telemetry := newTelemetry()
scannedAt := time.Date(2026, 4, 20, 16, 30, 0, 0, time.UTC) scannedAt := time.Date(2026, 4, 20, 16, 30, 0, 0, time.UTC)

View File

@ -103,6 +103,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
s.metrics.RecordPolicyBackup("excluded") s.metrics.RecordPolicyBackup("excluded")
continue continue
} }
// Only restic needs a second filesystem mount. Longhorn snapshots use
// the existing engine and must protect live RWO workloads too.
if s.cfg.BackupDriver == "restic" {
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes) blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
if err != nil { if err != nil {
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err) log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
@ -113,6 +116,7 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
s.metrics.RecordPolicyBackup("live_rwo_mount") s.metrics.RecordPolicyBackup("live_rwo_mount")
continue continue
} }
}
// Never enqueue a new policy backup while one is already active for this PVC. // Never enqueue a new policy backup while one is already active for this PVC.
// This prevents runaway job storms when a backup is stuck Pending/Running. // This prevents runaway job storms when a backup is stuck Pending/Running.
if pvc.ActiveBackups > 0 { if pvc.ActiveBackups > 0 {