package server import ( "context" "errors" "strings" "testing" "time" "scm.bstein.dev/bstein/soteria/internal/api" "scm.bstein.dev/bstein/soteria/internal/config" "scm.bstein.dev/bstein/soteria/internal/k8s" "scm.bstein.dev/bstein/soteria/internal/longhorn" ) type policyCycleTestKubeClient struct { *inventoryTestKubeClient createBackupErrForPVC map[string]error backupRequests []api.BackupRequest } func (k *policyCycleTestKubeClient) CreateBackupJob(ctx context.Context, cfg *config.Config, req api.BackupRequest) (string, string, error) { k.backupRequests = append(k.backupRequests, req) if err := k.createBackupErrForPVC[req.PVC]; err != nil { return "", "", err } return k.inventoryTestKubeClient.fakeKubeClient.CreateBackupJob(ctx, cfg, req) } func metricCount(samples map[string]metricSample, labels map[string]string) float64 { sample, ok := samples[metricKey(labels)] if !ok { return 0 } return sample.value } func findBackupRequestByPVC(items []api.BackupRequest, pvc string) (api.BackupRequest, bool) { for _, item := range items { if item.PVC == pvc { return item, true } } return api.BackupRequest{}, false } func TestRunPolicyCycleCoversInventoryErrorAndConcurrentGuard(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, listPVCsErr: errors.New("inventory exploded"), }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, PolicyEvalInterval: time.Minute, }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 { t.Fatalf("expected inventory_error policy metric, got %f", got) } if srv.running { t.Fatalf("expected policy runner lock to be released after inventory failure") } srv.running = true srv.runPolicyCycle(context.Background()) if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 { t.Fatalf("expected concurrent guard to skip second run, got %f", got) } } func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) { now := time.Now().UTC() recent := now.Add(-30 * time.Minute) client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"}, {Namespace: "apps", Name: "busy", VolumeName: "vol-busy", Phase: "Bound"}, {Namespace: "apps", Name: "recent", VolumeName: "vol-recent", Phase: "Bound"}, {Namespace: "apps", Name: "attempted", VolumeName: "vol-attempted", Phase: "Bound"}, {Namespace: "apps", Name: "fail", VolumeName: "vol-fail", Phase: "Bound"}, }, backupJobs: map[string][]k8s.BackupJobSummary{ "apps/busy": { {Name: "job-busy", Namespace: "apps", PVC: "busy", Repository: "s3:https://repo/root/isolated/apps/busy", State: "Running", CreatedAt: recent}, }, "apps/recent": { {Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1}, }, "apps/attempted": { {Name: "job-attempted", Namespace: "apps", PVC: "attempted", State: "Failed", CreatedAt: recent, KeepLast: 1}, }, }, }, }, createBackupErrForPVC: map[string]error{ "fail": errors.New("create backup job exploded"), }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ResticRepository: "s3:https://repo/root", }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true, KeepLast: 1}, "apps__data": {ID: "apps__data", Namespace: "apps", PVC: "data", IntervalHours: 1, Enabled: true, Dedupe: false, KeepLast: 2}, "apps__disabled": {ID: "apps__disabled", Namespace: "apps", PVC: "disabled", IntervalHours: 1, Enabled: false, Dedupe: false, KeepLast: 5}, }, } srv.runPolicyCycle(context.Background()) if len(client.backupRequests) != 2 { t.Fatalf("expected two executed backup requests, got %#v", client.backupRequests) } dataReq, ok := findBackupRequestByPVC(client.backupRequests, "data") if !ok || dataReq.Dedupe == nil || *dataReq.Dedupe != false || dataReq.KeepLast == nil || *dataReq.KeepLast != 2 { t.Fatalf("expected stricter PVC policy to win for data, got %#v", client.backupRequests) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 { t.Fatalf("expected one successful policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "backend_error"}); got != 1 { t.Fatalf("expected one backend error policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "in_progress"}); got != 1 { t.Fatalf("expected one in-progress policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "not_due"}); got != 2 { t.Fatalf("expected two not-due policy backups, got %f", got) } if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "success"}); got != 1 { t.Fatalf("expected one successful executed backup request, got %f", got) } if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "backend_error"}); got != 1 { t.Fatalf("expected one failed executed backup request, got %f", got) } } func TestRunPolicyCycleHonorsBackupLimit(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "alpha", VolumeName: "vol-alpha", Phase: "Bound"}, {Namespace: "apps", Name: "beta", VolumeName: "vol-beta", Phase: "Bound"}, }, }, }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ResticRepository: "s3:https://repo/root", PolicyBackupsPerCycle: 1, }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if len(client.backupRequests) != 1 { t.Fatalf("expected one policy backup due to cycle limit, got %#v", client.backupRequests) } if client.backupRequests[0].PVC != "alpha" { t.Fatalf("expected deterministic first backup for alpha, got %#v", client.backupRequests[0]) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "cycle_limit"}); got != 1 { t.Fatalf("expected cycle_limit metric, got %f", got) } } func TestRunPolicyCycleSkipsExcludedPVCsAndBusyRepositories(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "astreae"}, {Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"}, }, backupJobs: map[string][]k8s.BackupJobSummary{ "apps/other": { {Name: "job-other", Namespace: "apps", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()}, }, }, }, }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ResticRepository: "s3:https://repo/root", ExcludedStorageClasses: []string{"local-path"}, }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if len(client.backupRequests) != 0 { t.Fatalf("expected no backup requests while shared repo is busy and cache is excluded, got %#v", client.backupRequests) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "repo_in_progress"}); got != 1 { t.Fatalf("expected repo_in_progress metric, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "excluded"}); got != 1 { t.Fatalf("expected excluded metric, got %f", got) } } func TestRunPolicyCycleSkipsLiveExclusivePVCs(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}, {Namespace: "apps", Name: "shared", VolumeName: "vol-shared", Phase: "Bound", AccessModes: []string{"ReadWriteMany"}}, }, pvcMounts: map[string][]k8s.PVCMount{ "apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}}, "apps/shared": {{PodName: "web-0", NodeName: "titan-07", Phase: "Running"}}, }, }, }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ResticRepository: "s3:https://repo/root", }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if len(client.backupRequests) != 1 || client.backupRequests[0].PVC != "shared" { t.Fatalf("expected only RWX pvc backup request, got %#v", client.backupRequests) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "live_rwo_mount"}); got != 1 { t.Fatalf("expected live_rwo_mount skip metric, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 { t.Fatalf("expected one successful policy backup, got %f", got) } } func TestRunPolicyCycleSkipsDetachedLonghornVolumesWithoutConsumingLimit(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "detached", VolumeName: "vol-detached", Phase: "Bound"}, {Namespace: "apps", Name: "ready", VolumeName: "vol-ready", Phase: "Bound"}, }, }, }, } longhornClient := &fakeLonghornClient{ volumes: map[string]longhorn.Volume{ "vol-detached": {Name: "vol-detached", State: "detached"}, "vol-ready": {Name: "vol-ready", State: "attached"}, }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, PolicyBackupsPerCycle: 1, }, client: client, longhorn: longhornClient, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if longhornClient.createSnapshotName == "" || !strings.Contains(longhornClient.createSnapshotName, "apps-ready") { t.Fatalf("expected ready volume snapshot backup after detached skip, got snapshot %q", longhornClient.createSnapshotName) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "volume_not_ready"}); got != 1 { t.Fatalf("expected volume_not_ready metric, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 { t.Fatalf("expected one successful policy backup, got %f", got) } }