backup: allow live Longhorn policies and enforce exclusions
This commit is contained in:
parent
47205db5b2
commit
7018d4ca41
@ -206,10 +206,19 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
|
||||
|
||||
switch s.cfg.BackupDriver {
|
||||
case "longhorn":
|
||||
volumeName, _, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||
volumeName, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||
if err != nil {
|
||||
return api.BackupResponse{}, "validation_error", err
|
||||
}
|
||||
storageClass := ""
|
||||
if pvc != nil && pvc.Spec.StorageClassName != nil {
|
||||
storageClass = *pvc.Spec.StorageClassName
|
||||
}
|
||||
// Apply data-location exclusions to manual and namespace requests as
|
||||
// well as scheduled backups, before invoking the Longhorn backend.
|
||||
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
|
||||
return api.BackupResponse{}, "validation_error", errors.New(reason)
|
||||
}
|
||||
|
||||
backupID := backupName("backup", req.Namespace+"-"+req.PVC)
|
||||
response := api.BackupResponse{
|
||||
|
||||
54
internal/server/longhorn_live_policy_test.go
Normal file
54
internal/server/longhorn_live_policy_test.go
Normal file
@ -0,0 +1,54 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"scm.bstein.dev/bstein/soteria/internal/api"
|
||||
"scm.bstein.dev/bstein/soteria/internal/config"
|
||||
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
||||
)
|
||||
|
||||
// TestLiveRWOLonghornPolicy protects live workloads without a second mount.
|
||||
func TestLiveRWOLonghornPolicy(t *testing.T) {
|
||||
client := &policyCycleTestKubeClient{
|
||||
inventoryTestKubeClient: &inventoryTestKubeClient{
|
||||
fakeKubeClient: &fakeKubeClient{
|
||||
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
|
||||
pvcMounts: map[string][]k8s.PVCMount{"apps/data": {{PodName: "database-0", NodeName: "worker", Phase: "Running"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
backend := &fakeLonghornClient{}
|
||||
srv := &Server{
|
||||
cfg: &config.Config{BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1},
|
||||
client: client, longhorn: backend, metrics: newTelemetry(),
|
||||
policies: map[string]api.BackupPolicy{"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}},
|
||||
}
|
||||
srv.runPolicyCycle(context.Background())
|
||||
if backend.createSnapshotName == "" {
|
||||
t.Fatal("live RWO workload did not receive a Longhorn snapshot")
|
||||
}
|
||||
if len(client.backupRequests) != 0 {
|
||||
t.Fatal("Longhorn policy attempted to launch a filesystem-mount job")
|
||||
}
|
||||
if metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}) != 1 {
|
||||
t.Fatal("expected one successful policy backup")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLonghornExclusionAppliesToManualRequests prevents bypassing local-only policy.
|
||||
func TestLonghornExclusionAppliesToManualRequests(t *testing.T) {
|
||||
backend := &fakeLonghornClient{}
|
||||
srv := &Server{
|
||||
cfg: &config.Config{BackupDriver: "longhorn", ExcludedPVCs: []string{"hermes/*"}},
|
||||
client: &fakeKubeClient{}, longhorn: backend,
|
||||
}
|
||||
for _, dryRun := range []bool{false, true} {
|
||||
_, code, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "hermes", PVC: "workspace", DryRun: dryRun}, "test")
|
||||
if err == nil || code != "validation_error" || backend.createSnapshotName != "" {
|
||||
t.Fatal("excluded Longhorn PVC reached the backup backend")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -103,15 +103,19 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
||||
s.metrics.RecordPolicyBackup("excluded")
|
||||
continue
|
||||
}
|
||||
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||
if err != nil {
|
||||
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||
continue
|
||||
}
|
||||
if blocked {
|
||||
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||
continue
|
||||
// Only restic needs a second filesystem mount. Longhorn snapshots use
|
||||
// the existing engine and must protect live RWO workloads too.
|
||||
if s.cfg.BackupDriver == "restic" {
|
||||
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||
if err != nil {
|
||||
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||
continue
|
||||
}
|
||||
if blocked {
|
||||
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Never enqueue a new policy backup while one is already active for this PVC.
|
||||
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user