package server import ( "context" "errors" "testing" "time" "scm.bstein.dev/bstein/soteria/internal/api" "scm.bstein.dev/bstein/soteria/internal/config" "scm.bstein.dev/bstein/soteria/internal/k8s" ) type policyCycleTestKubeClient struct { *inventoryTestKubeClient createBackupErrForPVC map[string]error backupRequests []api.BackupRequest } func (k *policyCycleTestKubeClient) CreateBackupJob(ctx context.Context, cfg *config.Config, req api.BackupRequest) (string, string, error) { k.backupRequests = append(k.backupRequests, req) if err := k.createBackupErrForPVC[req.PVC]; err != nil { return "", "", err } return k.inventoryTestKubeClient.fakeKubeClient.CreateBackupJob(ctx, cfg, req) } func metricCount(samples map[string]metricSample, labels map[string]string) float64 { sample, ok := samples[metricKey(labels)] if !ok { return 0 } return sample.value } func findBackupRequestByPVC(items []api.BackupRequest, pvc string) (api.BackupRequest, bool) { for _, item := range items { if item.PVC == pvc { return item, true } } return api.BackupRequest{}, false } func TestRunPolicyCycleCoversInventoryErrorAndConcurrentGuard(t *testing.T) { client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, listPVCsErr: errors.New("inventory exploded"), }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, PolicyEvalInterval: time.Minute, }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true}, }, } srv.runPolicyCycle(context.Background()) if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 { t.Fatalf("expected inventory_error policy metric, got %f", got) } if srv.running { t.Fatalf("expected policy runner lock to be released after inventory failure") } srv.running = true srv.runPolicyCycle(context.Background()) if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 { t.Fatalf("expected concurrent guard to skip second run, got %f", got) } } func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) { now := time.Now().UTC() recent := now.Add(-30 * time.Minute) client := &policyCycleTestKubeClient{ inventoryTestKubeClient: &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"}, {Namespace: "apps", Name: "busy", VolumeName: "vol-busy", Phase: "Bound"}, {Namespace: "apps", Name: "recent", VolumeName: "vol-recent", Phase: "Bound"}, {Namespace: "apps", Name: "attempted", VolumeName: "vol-attempted", Phase: "Bound"}, {Namespace: "apps", Name: "fail", VolumeName: "vol-fail", Phase: "Bound"}, }, backupJobs: map[string][]k8s.BackupJobSummary{ "apps/busy": { {Name: "job-busy", Namespace: "apps", PVC: "busy", State: "Running", CreatedAt: recent}, }, "apps/recent": { {Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1}, }, "apps/attempted": { {Name: "job-attempted", Namespace: "apps", PVC: "attempted", State: "Failed", CreatedAt: recent, KeepLast: 1}, }, }, }, }, createBackupErrForPVC: map[string]error{ "fail": errors.New("create backup job exploded"), }, } srv := &Server{ cfg: &config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ResticRepository: "s3:https://repo/root", }, client: client, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{ "apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true, KeepLast: 1}, "apps__data": {ID: "apps__data", Namespace: "apps", PVC: "data", IntervalHours: 1, Enabled: true, Dedupe: false, KeepLast: 2}, "apps__disabled": {ID: "apps__disabled", Namespace: "apps", PVC: "disabled", IntervalHours: 1, Enabled: false, Dedupe: false, KeepLast: 5}, }, } srv.runPolicyCycle(context.Background()) if len(client.backupRequests) != 2 { t.Fatalf("expected two executed backup requests, got %#v", client.backupRequests) } dataReq, ok := findBackupRequestByPVC(client.backupRequests, "data") if !ok || dataReq.Dedupe == nil || *dataReq.Dedupe != false || dataReq.KeepLast == nil || *dataReq.KeepLast != 2 { t.Fatalf("expected stricter PVC policy to win for data, got %#v", client.backupRequests) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 { t.Fatalf("expected one successful policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "backend_error"}); got != 1 { t.Fatalf("expected one backend error policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "in_progress"}); got != 1 { t.Fatalf("expected one in-progress policy backup, got %f", got) } if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "not_due"}); got != 2 { t.Fatalf("expected two not-due policy backups, got %f", got) } if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "success"}); got != 1 { t.Fatalf("expected one successful executed backup request, got %f", got) } if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "backend_error"}); got != 1 { t.Fatalf("expected one failed executed backup request, got %f", got) } }