backup: allow live Longhorn policies and enforce exclusions
This commit is contained in:
parent
47205db5b2
commit
7018d4ca41
@ -206,10 +206,19 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
|
|||||||
|
|
||||||
switch s.cfg.BackupDriver {
|
switch s.cfg.BackupDriver {
|
||||||
case "longhorn":
|
case "longhorn":
|
||||||
volumeName, _, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
volumeName, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.BackupResponse{}, "validation_error", err
|
return api.BackupResponse{}, "validation_error", err
|
||||||
}
|
}
|
||||||
|
storageClass := ""
|
||||||
|
if pvc != nil && pvc.Spec.StorageClassName != nil {
|
||||||
|
storageClass = *pvc.Spec.StorageClassName
|
||||||
|
}
|
||||||
|
// Apply data-location exclusions to manual and namespace requests as
|
||||||
|
// well as scheduled backups, before invoking the Longhorn backend.
|
||||||
|
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
|
||||||
|
return api.BackupResponse{}, "validation_error", errors.New(reason)
|
||||||
|
}
|
||||||
|
|
||||||
backupID := backupName("backup", req.Namespace+"-"+req.PVC)
|
backupID := backupName("backup", req.Namespace+"-"+req.PVC)
|
||||||
response := api.BackupResponse{
|
response := api.BackupResponse{
|
||||||
|
|||||||
54
internal/server/longhorn_live_policy_test.go
Normal file
54
internal/server/longhorn_live_policy_test.go
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"scm.bstein.dev/bstein/soteria/internal/api"
|
||||||
|
"scm.bstein.dev/bstein/soteria/internal/config"
|
||||||
|
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestLiveRWOLonghornPolicy protects live workloads without a second mount.
|
||||||
|
func TestLiveRWOLonghornPolicy(t *testing.T) {
|
||||||
|
client := &policyCycleTestKubeClient{
|
||||||
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
||||||
|
fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
|
||||||
|
pvcMounts: map[string][]k8s.PVCMount{"apps/data": {{PodName: "database-0", NodeName: "worker", Phase: "Running"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
backend := &fakeLonghornClient{}
|
||||||
|
srv := &Server{
|
||||||
|
cfg: &config.Config{BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1},
|
||||||
|
client: client, longhorn: backend, metrics: newTelemetry(),
|
||||||
|
policies: map[string]api.BackupPolicy{"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}},
|
||||||
|
}
|
||||||
|
srv.runPolicyCycle(context.Background())
|
||||||
|
if backend.createSnapshotName == "" {
|
||||||
|
t.Fatal("live RWO workload did not receive a Longhorn snapshot")
|
||||||
|
}
|
||||||
|
if len(client.backupRequests) != 0 {
|
||||||
|
t.Fatal("Longhorn policy attempted to launch a filesystem-mount job")
|
||||||
|
}
|
||||||
|
if metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}) != 1 {
|
||||||
|
t.Fatal("expected one successful policy backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLonghornExclusionAppliesToManualRequests prevents bypassing local-only policy.
|
||||||
|
func TestLonghornExclusionAppliesToManualRequests(t *testing.T) {
|
||||||
|
backend := &fakeLonghornClient{}
|
||||||
|
srv := &Server{
|
||||||
|
cfg: &config.Config{BackupDriver: "longhorn", ExcludedPVCs: []string{"hermes/*"}},
|
||||||
|
client: &fakeKubeClient{}, longhorn: backend,
|
||||||
|
}
|
||||||
|
for _, dryRun := range []bool{false, true} {
|
||||||
|
_, code, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "hermes", PVC: "workspace", DryRun: dryRun}, "test")
|
||||||
|
if err == nil || code != "validation_error" || backend.createSnapshotName != "" {
|
||||||
|
t.Fatal("excluded Longhorn PVC reached the backup backend")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -103,15 +103,19 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
s.metrics.RecordPolicyBackup("excluded")
|
s.metrics.RecordPolicyBackup("excluded")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
// Only restic needs a second filesystem mount. Longhorn snapshots use
|
||||||
if err != nil {
|
// the existing engine and must protect live RWO workloads too.
|
||||||
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
if s.cfg.BackupDriver == "restic" {
|
||||||
s.metrics.RecordPolicyBackup("active_lookup_error")
|
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||||
continue
|
if err != nil {
|
||||||
}
|
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||||
if blocked {
|
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||||
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
continue
|
||||||
continue
|
}
|
||||||
|
if blocked {
|
||||||
|
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Never enqueue a new policy backup while one is already active for this PVC.
|
// Never enqueue a new policy backup while one is already active for this PVC.
|
||||||
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user