backup: allow live Longhorn policies and enforce exclusions

This commit is contained in:
codex 2026-10-03 00:26:20 -05:00
parent 47205db5b2
commit 7018d4ca41
3 changed files with 77 additions and 10 deletions

View File

@ -206,10 +206,19 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
switch s.cfg.BackupDriver {
case "longhorn":
volumeName, _, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
volumeName, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
if err != nil {
return api.BackupResponse{}, "validation_error", err
}
storageClass := ""
if pvc != nil && pvc.Spec.StorageClassName != nil {
storageClass = *pvc.Spec.StorageClassName
}
// Apply data-location exclusions to manual and namespace requests as
// well as scheduled backups, before invoking the Longhorn backend.
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
backupID := backupName("backup", req.Namespace+"-"+req.PVC)
response := api.BackupResponse{

View File

@ -0,0 +1,54 @@
package server
import (
"context"
"testing"
"time"
"scm.bstein.dev/bstein/soteria/internal/api"
"scm.bstein.dev/bstein/soteria/internal/config"
"scm.bstein.dev/bstein/soteria/internal/k8s"
)
// TestLiveRWOLonghornPolicy protects live workloads without a second mount.
func TestLiveRWOLonghornPolicy(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
pvcMounts: map[string][]k8s.PVCMount{"apps/data": {{PodName: "database-0", NodeName: "worker", Phase: "Running"}}},
},
},
}
backend := &fakeLonghornClient{}
srv := &Server{
cfg: &config.Config{BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1},
client: client, longhorn: backend, metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}},
}
srv.runPolicyCycle(context.Background())
if backend.createSnapshotName == "" {
t.Fatal("live RWO workload did not receive a Longhorn snapshot")
}
if len(client.backupRequests) != 0 {
t.Fatal("Longhorn policy attempted to launch a filesystem-mount job")
}
if metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}) != 1 {
t.Fatal("expected one successful policy backup")
}
}
// TestLonghornExclusionAppliesToManualRequests prevents bypassing local-only policy.
func TestLonghornExclusionAppliesToManualRequests(t *testing.T) {
backend := &fakeLonghornClient{}
srv := &Server{
cfg: &config.Config{BackupDriver: "longhorn", ExcludedPVCs: []string{"hermes/*"}},
client: &fakeKubeClient{}, longhorn: backend,
}
for _, dryRun := range []bool{false, true} {
_, code, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "hermes", PVC: "workspace", DryRun: dryRun}, "test")
if err == nil || code != "validation_error" || backend.createSnapshotName != "" {
t.Fatal("excluded Longhorn PVC reached the backup backend")
}
}
}

View File

@ -103,15 +103,19 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
s.metrics.RecordPolicyBackup("excluded")
continue
}
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
if err != nil {
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
s.metrics.RecordPolicyBackup("active_lookup_error")
continue
}
if blocked {
s.metrics.RecordPolicyBackup("live_rwo_mount")
continue
// Only restic needs a second filesystem mount. Longhorn snapshots use
// the existing engine and must protect live RWO workloads too.
if s.cfg.BackupDriver == "restic" {
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
if err != nil {
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
s.metrics.RecordPolicyBackup("active_lookup_error")
continue
}
if blocked {
s.metrics.RecordPolicyBackup("live_rwo_mount")
continue
}
}
// Never enqueue a new policy backup while one is already active for this PVC.
// This prevents runaway job storms when a backup is stuck Pending/Running.