package server import ( "context" "errors" "net/http" "net/http/httptest" "strings" "testing" "scm.bstein.dev/bstein/soteria/internal/api" "scm.bstein.dev/bstein/soteria/internal/config" "scm.bstein.dev/bstein/soteria/internal/k8s" "scm.bstein.dev/bstein/soteria/internal/longhorn" corev1 "k8s.io/api/core/v1" ) type restoreTestKubeClient struct { *fakeKubeClient listPVCsErr error targetExistsErr error resolveErr error createRestoreErr error } func (k *restoreTestKubeClient) ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error) { if k.listPVCsErr != nil { return nil, k.listPVCsErr } return k.fakeKubeClient.ListBoundPVCs(ctx) } func (k *restoreTestKubeClient) PersistentVolumeClaimExists(ctx context.Context, namespace, pvcName string) (bool, error) { if k.targetExistsErr != nil { return false, k.targetExistsErr } return k.fakeKubeClient.PersistentVolumeClaimExists(ctx, namespace, pvcName) } func (k *restoreTestKubeClient) ResolvePVCVolume(ctx context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) { if k.resolveErr != nil { return "", nil, nil, k.resolveErr } return k.fakeKubeClient.ResolvePVCVolume(ctx, namespace, pvcName) } func (k *restoreTestKubeClient) CreateRestoreJob(ctx context.Context, cfg *config.Config, req api.RestoreTestRequest) (string, string, error) { if k.createRestoreErr != nil { return "", "", k.createRestoreErr } return k.fakeKubeClient.CreateRestoreJob(ctx, cfg, req) } type restoreTestLonghornClient struct { *fakeLonghornClient findBackup *longhorn.Backup findErr error getVolume *longhorn.Volume getVolumeErr error createVolErr error createPVCErr error deleteVolErr error deletedVolume string } func (l *restoreTestLonghornClient) FindBackup(ctx context.Context, volumeName, snapshot string) (*longhorn.Backup, error) { if l.findErr != nil { return nil, l.findErr } if l.findBackup != nil { return l.findBackup, nil } return l.fakeLonghornClient.FindBackup(ctx, volumeName, snapshot) } func (l *restoreTestLonghornClient) GetVolume(ctx context.Context, volume string) (*longhorn.Volume, error) { if l.getVolumeErr != nil { return nil, l.getVolumeErr } if l.getVolume != nil { return l.getVolume, nil } return l.fakeLonghornClient.GetVolume(ctx, volume) } func (l *restoreTestLonghornClient) CreateVolumeFromBackup(ctx context.Context, name, size string, replicas int, backupURL string) (*longhorn.Volume, error) { if l.createVolErr != nil { return nil, l.createVolErr } return l.fakeLonghornClient.CreateVolumeFromBackup(ctx, name, size, replicas, backupURL) } func (l *restoreTestLonghornClient) CreatePVC(ctx context.Context, volumeName, namespace, pvcName string) error { if l.createPVCErr != nil { return l.createPVCErr } return l.fakeLonghornClient.CreatePVC(ctx, volumeName, namespace, pvcName) } func (l *restoreTestLonghornClient) DeleteVolume(ctx context.Context, volumeName string) error { l.deletedVolume = volumeName if l.deleteVolErr != nil { return l.deleteVolErr } return l.fakeLonghornClient.DeleteVolume(ctx, volumeName) } func newRestoreTestServer(cfg *config.Config, client kubeClient, longhornClient longhornClient) *Server { srv := &Server{ cfg: cfg, client: client, longhorn: longhornClient, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{}, } srv.handler = http.HandlerFunc(srv.route) return srv } func TestHandleRestoreRejectsInvalidRequests(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) testCases := []struct { name string method string body string status int want string }{ {name: "method", method: http.MethodGet, body: "", status: http.StatusMethodNotAllowed, want: "method not allowed"}, {name: "invalid json", method: http.MethodPost, body: `{"namespace":`, status: http.StatusBadRequest, want: "invalid JSON"}, {name: "missing namespace", method: http.MethodPost, body: `{"pvc":"data","target_pvc":"restore-data"}`, status: http.StatusBadRequest, want: "namespace is required"}, {name: "missing pvc", method: http.MethodPost, body: `{"namespace":"apps","target_pvc":"restore-data"}`, status: http.StatusBadRequest, want: "pvc is required"}, {name: "missing target pvc", method: http.MethodPost, body: `{"namespace":"apps","pvc":"data"}`, status: http.StatusBadRequest, want: "target_pvc is required"}, } for _, tc := range testCases { req := httptest.NewRequest(tc.method, "/v1/restores", strings.NewReader(tc.body)) if tc.method == http.MethodPost { req.Header.Set("Content-Type", "application/json") } res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != tc.status || !strings.Contains(res.Body.String(), tc.want) { t.Fatalf("%s: expected %d/%q, got %d %s", tc.name, tc.status, tc.want, res.Code, res.Body.String()) } } } func TestHandleRestoreCoversAdditionalValidationBranches(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) testCases := []struct { name string body string want string code int }{ { name: "invalid namespace label", body: `{"namespace":"Bad_Ns","pvc":"data","target_pvc":"restore-data"}`, want: "namespace must be a valid Kubernetes DNS-1123 label", code: http.StatusBadRequest, }, { name: "invalid pvc label", body: `{"namespace":"apps","pvc":"Bad_PVC","target_pvc":"restore-data"}`, want: "pvc must be a valid Kubernetes DNS-1123 label", code: http.StatusBadRequest, }, { name: "invalid target pvc label", body: `{"namespace":"apps","pvc":"data","target_pvc":"Bad_Target"}`, want: "target_pvc must be a valid Kubernetes DNS-1123 label", code: http.StatusBadRequest, }, { name: "default target namespace still conflicts", body: `{"namespace":"apps","pvc":"data","target_pvc":"data"}`, want: "target namespace/pvc must differ from source", code: http.StatusConflict, }, } for _, tc := range testCases { req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(tc.body)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != tc.code || !strings.Contains(res.Body.String(), tc.want) { t.Fatalf("%s: expected %d/%q, got %d %s", tc.name, tc.code, tc.want, res.Code, res.Body.String()) } } } func TestHandleNamespaceRestoreRejectsInvalidRequestsAndBackendErrors(t *testing.T) { t.Run("bad requests", func(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) testCases := []struct { name string method string body string status int want string }{ {name: "method", method: http.MethodGet, body: "", status: http.StatusMethodNotAllowed, want: "method not allowed"}, {name: "invalid json", method: http.MethodPost, body: `{"namespace":`, status: http.StatusBadRequest, want: "invalid JSON"}, {name: "missing namespace", method: http.MethodPost, body: `{}`, status: http.StatusBadRequest, want: "namespace is required"}, {name: "invalid namespace", method: http.MethodPost, body: `{"namespace":"Bad_Ns"}`, status: http.StatusBadRequest, want: "namespace must be a valid Kubernetes DNS-1123 label"}, {name: "invalid target namespace", method: http.MethodPost, body: `{"namespace":"apps","target_namespace":"Bad_Ns"}`, status: http.StatusBadRequest, want: "target_namespace must be a valid Kubernetes DNS-1123 label"}, } for _, tc := range testCases { req := httptest.NewRequest(tc.method, "/v1/restores/namespace", strings.NewReader(tc.body)) if tc.method == http.MethodPost { req.Header.Set("Content-Type", "application/json") } res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != tc.status || !strings.Contains(res.Body.String(), tc.want) { t.Fatalf("%s: expected %d/%q, got %d %s", tc.name, tc.status, tc.want, res.Code, res.Body.String()) } } }) t.Run("backend error", func(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, listPVCsErr: errors.New("list pvcs exploded"), }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(`{"namespace":"apps"}`)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusBadGateway || !strings.Contains(res.Body.String(), "list pvcs exploded") { t.Fatalf("expected namespace restore backend error, got %d %s", res.Code, res.Body.String()) } }) } func TestHandleNamespaceRestoreDefaultsTargetNamespaceForEmptyResults(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "other", Name: "cache", VolumeName: "pv-other-cache", Phase: "Bound"}, }, }, }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(`{"namespace":"apps","dry_run":false}`)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } if !strings.Contains(res.Body.String(), `"target_namespace":"apps"`) || !strings.Contains(res.Body.String(), `"total":0`) { t.Fatalf("expected default target namespace and empty results, got %s", res.Body.String()) } } func TestHandleNamespaceRestoreAggregatesFailures(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "cache", VolumeName: "pv-apps-cache", Phase: "Bound"}, }, targetExists: true, }, }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(`{"namespace":"apps","target_namespace":"restore"}`)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } if !strings.Contains(res.Body.String(), `"failed":1`) || !strings.Contains(res.Body.String(), `"status":"conflict"`) { t.Fatalf("expected aggregated failure response, got %s", res.Body.String()) } } func TestExecuteRestoreLonghornErrorAndCleanupPaths(t *testing.T) { cfg := &config.Config{AuthRequired: false, BackupDriver: "longhorn"} t.Run("target exists lookup error", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, targetExistsErr: errors.New("target lookup exploded"), }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "target lookup exploded") { t.Fatalf("expected target exists validation error, got result=%q err=%v", result, err) } }) t.Run("resolve pvc error", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, resolveErr: errors.New("resolve pvc exploded"), }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "resolve pvc exploded") { t.Fatalf("expected resolve pvc validation error, got result=%q err=%v", result, err) } }) t.Run("find backup error", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findErr: errors.New("find backup exploded"), }, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "find backup exploded") { t.Fatalf("expected find backup validation error, got result=%q err=%v", result, err) } }) t.Run("backup url required", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findBackup: &longhorn.Backup{Name: "backup-a", URL: ""}, }, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "backup_url is required") { t.Fatalf("expected backup_url validation error, got result=%q err=%v", result, err) } }) t.Run("get volume backend error", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findBackup: &longhorn.Backup{Name: "backup-a", URL: "s3://bucket/backup-a"}, getVolumeErr: errors.New("get volume exploded"), }, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "backend_error" || err == nil || !strings.Contains(err.Error(), "get volume exploded") { t.Fatalf("expected get volume backend error, got result=%q err=%v", result, err) } }) t.Run("create volume backend error", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findBackup: &longhorn.Backup{Name: "backup-a", URL: "s3://bucket/backup-a"}, getVolume: &longhorn.Volume{Name: "pv-apps-data", Size: "4096", NumberOfReplicas: 0}, createVolErr: errors.New("create volume exploded"), }, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "backend_error" || err == nil || !strings.Contains(err.Error(), "create volume exploded") { t.Fatalf("expected create volume backend error, got result=%q err=%v", result, err) } }) t.Run("create pvc cleanup success", func(t *testing.T) { longhornClient := &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findBackup: &longhorn.Backup{Name: "backup-a", URL: "s3://bucket/backup-a"}, getVolume: &longhorn.Volume{Name: "pv-apps-data", Size: "4096", NumberOfReplicas: 1}, createPVCErr: errors.New("create pvc exploded"), } srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, longhornClient, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "backend_error" || err == nil || !strings.Contains(err.Error(), "create restore pvc") { t.Fatalf("expected create pvc backend error, got result=%q err=%v", result, err) } if longhornClient.deletedVolume == "" { t.Fatalf("expected failed PVC create to trigger volume cleanup") } }) t.Run("create pvc cleanup failure", func(t *testing.T) { srv := newRestoreTestServer( cfg, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, findBackup: &longhorn.Backup{Name: "backup-a", URL: "s3://bucket/backup-a"}, getVolume: &longhorn.Volume{Name: "pv-apps-data", Size: "4096", NumberOfReplicas: 1}, createPVCErr: errors.New("create pvc exploded"), deleteVolErr: errors.New("cleanup exploded"), }, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "backend_error" || err == nil || !strings.Contains(err.Error(), "cleanup failed") { t.Fatalf("expected cleanup failure to be surfaced, got result=%q err=%v", result, err) } }) } func TestExecuteRestoreResticAndHelperFunctions(t *testing.T) { t.Run("restic create restore error", func(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "restic"}, &restoreTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, createRestoreErr: errors.New("create restore job exploded"), }, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "backend_error" || err == nil || !strings.Contains(err.Error(), "create restore job exploded") { t.Fatalf("expected restic backend error, got result=%q err=%v", result, err) } }) t.Run("unsupported driver", func(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "mystery"}, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) _, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", }, "brad") if result != "unsupported_driver" || err == nil || !strings.Contains(err.Error(), "unsupported backup driver") { t.Fatalf("expected unsupported driver error, got result=%q err=%v", result, err) } }) t.Run("restic dry run success", func(t *testing.T) { srv := newRestoreTestServer( &config.Config{AuthRequired: false, BackupDriver: "restic"}, &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}, ) response, result, err := srv.executeRestore(context.Background(), api.RestoreTestRequest{ Namespace: "apps", PVC: "data", TargetNamespace: "restore", TargetPVC: "restore-data", DryRun: true, }, "brad") if err != nil || result != "dry_run" || response.JobName != "restore-job" || response.Secret != "restore-secret" { t.Fatalf("expected restic dry-run success, got response=%#v result=%q err=%v", response, result, err) } }) t.Run("restore status code", func(t *testing.T) { testCases := map[string]int{ "validation_error": http.StatusBadRequest, "unsupported_driver": http.StatusBadRequest, "conflict": http.StatusConflict, "backend_error": http.StatusBadGateway, "anything-else": http.StatusInternalServerError, } for result, want := range testCases { if got := restoreStatusCode(result); got != want { t.Fatalf("result=%q: expected %d, got %d", result, want, got) } } }) t.Run("target pvc name", func(t *testing.T) { if got := targetPVCName("", "Data_PVC"); got != "restore-data-pvc" { t.Fatalf("expected default restore prefix, got %q", got) } if got := targetPVCName("___", "Data_PVC"); got != "restore-data-pvc" { t.Fatalf("expected sanitized-empty prefix to fall back to restore, got %q", got) } if got := targetPVCName("drill", "cache"); got != "drill-cache" { t.Fatalf("expected explicit prefix, got %q", got) } if got := targetPVCName("drill-", "cache"); got != "drill-cache" { t.Fatalf("expected suffix-preserving prefix, got %q", got) } if got := targetPVCName(strings.Repeat("very-long-prefix-", 8), strings.Repeat("data-", 8)); len(got) > 63 { t.Fatalf("expected truncated target pvc name <=63 chars, got %d %q", len(got), got) } }) }