package server import ( "context" "encoding/json" "net/http" "net/http/httptest" "net/url" "strings" "testing" "time" "scm.bstein.dev/bstein/soteria/internal/api" "scm.bstein.dev/bstein/soteria/internal/config" "scm.bstein.dev/bstein/soteria/internal/k8s" "scm.bstein.dev/bstein/soteria/internal/longhorn" corev1 "k8s.io/api/core/v1" ) type fakeKubeClient struct { pvcs []k8s.PVCSummary backupJobs map[string][]k8s.BackupJobSummary targetExists bool secretData map[string][]byte } func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) { return namespace + "-" + pvcName + "-pv", nil, nil, nil } func (f *fakeKubeClient) CreateBackupJob(_ context.Context, _ *config.Config, _ api.BackupRequest) (string, string, error) { return "backup-job", "backup-secret", nil } func (f *fakeKubeClient) CreateRestoreJob(_ context.Context, _ *config.Config, _ api.RestoreTestRequest) (string, string, error) { return "restore-job", "restore-secret", nil } func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, error) { return f.pvcs, nil } func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) { if f.backupJobs == nil { return nil, nil } key := namespace + "/" + pvc items := f.backupJobs[key] out := make([]k8s.BackupJobSummary, len(items)) copy(out, items) return out, nil } func (f *fakeKubeClient) PersistentVolumeClaimExists(_ context.Context, _, _ string) (bool, error) { return f.targetExists, nil } func (f *fakeKubeClient) LoadSecretData(_ context.Context, _, _, key string) ([]byte, error) { if f.secretData == nil { return nil, nil } value, ok := f.secretData[key] if !ok { return nil, nil } copyValue := make([]byte, len(value)) copy(copyValue, value) return copyValue, nil } func (f *fakeKubeClient) SaveSecretData(_ context.Context, _, _, key string, value []byte, _ map[string]string) error { if f.secretData == nil { f.secretData = map[string][]byte{} } copyValue := make([]byte, len(value)) copy(copyValue, value) f.secretData[key] = copyValue return nil } type fakeLonghornClient struct { backups []longhorn.Backup createSnapshotName string snapshotBackupName string } func (f *fakeLonghornClient) CreateSnapshot(_ context.Context, volume, name string, labels map[string]string) error { f.createSnapshotName = name return nil } func (f *fakeLonghornClient) SnapshotBackup(_ context.Context, volume, name string, labels map[string]string, backupMode string) (*longhorn.Volume, error) { f.snapshotBackupName = name return &longhorn.Volume{Name: volume}, nil } func (f *fakeLonghornClient) GetVolume(_ context.Context, volume string) (*longhorn.Volume, error) { return &longhorn.Volume{Name: volume, Size: "1073741824", NumberOfReplicas: 2}, nil } func (f *fakeLonghornClient) CreateVolumeFromBackup(_ context.Context, name, size string, replicas int, backupURL string) (*longhorn.Volume, error) { return &longhorn.Volume{Name: name, Size: size, NumberOfReplicas: replicas}, nil } func (f *fakeLonghornClient) CreatePVC(_ context.Context, volumeName, namespace, pvcName string) error { return nil } func (f *fakeLonghornClient) DeleteVolume(_ context.Context, volumeName string) error { return nil } func (f *fakeLonghornClient) FindBackup(_ context.Context, volumeName, snapshot string) (*longhorn.Backup, error) { return &longhorn.Backup{Name: "backup-latest", URL: "s3://bucket/backup-latest", State: "Completed"}, nil } func (f *fakeLonghornClient) ListBackups(_ context.Context, volumeName string) ([]longhorn.Backup, error) { return f.backups, nil } func TestProtectedInventoryRequiresAuth(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn"}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusUnauthorized { t.Fatalf("expected 401, got %d", res.Code) } } func TestProtectedInventoryAllowsMaintenanceGroup(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour}, client: &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}}, longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil) req.Header.Set("X-Auth-Request-User", "brad") req.Header.Set("X-Auth-Request-Groups", "/maintenance") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.InventoryResponse if err := json.NewDecoder(strings.NewReader(res.Body.String())).Decode(&payload); err != nil { t.Fatalf("decode inventory: %v", err) } if len(payload.Namespaces) != 1 || payload.Namespaces[0].Name != "apps" { t.Fatalf("unexpected inventory payload: %#v", payload) } } func TestProtectedInventoryAllowsForwardedHeaders(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour}, client: &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}}, longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil) req.Header.Set("X-Forwarded-User", "brad") req.Header.Set("X-Forwarded-Groups", "/ops;/maintenance") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } } func TestRestoreRejectsExistingTargetPVC(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, client: &fakeKubeClient{targetExists: true}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"restore-data"}` req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body)) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusConflict { t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String()) } } func TestRestoreRejectsInvalidTargetNamespace(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","pvc":"data","target_namespace":"Apps_Ns","target_pvc":"restore-data","backup_url":"s3://bucket/backup"}` req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body)) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusBadRequest { t.Fatalf("expected 400, got %d: %s", res.Code, res.Body.String()) } } func TestRestoreRejectsSameSourceAndTarget(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"data","backup_url":"s3://bucket/backup"}` req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body)) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusConflict { t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String()) } } func TestBackupCreatesSnapshotBeforeBackup(t *testing.T) { lh := &fakeLonghornClient{} srv := &Server{ cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn", LonghornBackupMode: "incremental"}, client: &fakeKubeClient{}, longhorn: lh, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","pvc":"data","dry_run":false}` req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(body)) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } if lh.createSnapshotName == "" { t.Fatalf("expected snapshot creation call") } if lh.snapshotBackupName == "" { t.Fatalf("expected snapshot backup call") } if lh.createSnapshotName != lh.snapshotBackupName { t.Fatalf("expected same snapshot and backup name, got snapshot=%q backup=%q", lh.createSnapshotName, lh.snapshotBackupName) } if !strings.HasPrefix(lh.createSnapshotName, "soteria-backup-apps-data-") { t.Fatalf("unexpected generated backup name %q", lh.createSnapshotName) } } func TestResticInventoryUsesCompletedBackupJobs(t *testing.T) { completedAt := time.Now().UTC().Add(-2 * time.Hour) srv := &Server{ cfg: &config.Config{ AuthRequired: false, BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, }, client: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}, }, backupJobs: map[string][]k8s.BackupJobSummary{ "apps/data": { { Name: "soteria-backup-data-20260413-000000", Namespace: "apps", PVC: "data", CreatedAt: completedAt.Add(-2 * time.Minute), CompletionTime: completedAt, State: "Completed", }, }, }, }, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.InventoryResponse if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil { t.Fatalf("decode inventory: %v", err) } if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 { t.Fatalf("unexpected inventory payload: %#v", payload) } entry := payload.Namespaces[0].PVCs[0] if !entry.Healthy { t.Fatalf("expected healthy entry, got %#v", entry) } if entry.HealthReason != "fresh" { t.Fatalf("expected fresh health reason, got %#v", entry.HealthReason) } if entry.CompletedBackups != 1 || entry.BackupCount != 1 { t.Fatalf("expected one completed backup, got %#v", entry) } } func TestResticInventoryMarksInProgressWhenOnlyActiveJobsExist(t *testing.T) { startedAt := time.Now().UTC().Add(-5 * time.Minute) srv := &Server{ cfg: &config.Config{ AuthRequired: false, BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, }, client: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}, }, backupJobs: map[string][]k8s.BackupJobSummary{ "apps/data": { { Name: "soteria-backup-data-20260413-010000", Namespace: "apps", PVC: "data", CreatedAt: startedAt, State: "Running", }, }, }, }, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.InventoryResponse if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil { t.Fatalf("decode inventory: %v", err) } if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 { t.Fatalf("unexpected inventory payload: %#v", payload) } entry := payload.Namespaces[0].PVCs[0] if entry.Healthy { t.Fatalf("expected in-progress backup to be unhealthy until completion, got %#v", entry) } if entry.HealthReason != "in_progress" { t.Fatalf("expected in_progress health reason, got %#v", entry.HealthReason) } if entry.ActiveBackups != 1 || entry.CompletedBackups != 0 { t.Fatalf("expected one active backup and zero completed backups, got %#v", entry) } if entry.LastJobName == "" || entry.LastJobState != "Running" { t.Fatalf("expected latest job metadata, got %#v", entry) } if entry.LastJobProgressPct != 70 { t.Fatalf("expected running progress to be 70, got %#v", entry.LastJobProgressPct) } } func TestResticBackupsEndpointReturnsLatestSelector(t *testing.T) { completedAt := time.Now().UTC().Add(-30 * time.Minute) srv := &Server{ cfg: &config.Config{ AuthRequired: false, BackupDriver: "restic", }, client: &fakeKubeClient{ backupJobs: map[string][]k8s.BackupJobSummary{ "apps/data": { { Name: "soteria-backup-data-20260413-010000", Namespace: "apps", PVC: "data", CreatedAt: completedAt.Add(-2 * time.Minute), CompletionTime: completedAt, State: "Completed", }, }, }, }, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/v1/backups?namespace=apps&pvc=data", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.BackupListResponse if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil { t.Fatalf("decode backups response: %v", err) } if len(payload.Backups) != 1 { t.Fatalf("expected one backup record, got %#v", payload.Backups) } if payload.Backups[0].URL != "latest" || !payload.Backups[0].Latest { t.Fatalf("expected latest restic selector, got %#v", payload.Backups[0]) } } func TestMetricsStayPublic(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin"}}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/metrics", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d", res.Code) } if !strings.Contains(res.Body.String(), "soteria_backup_requests_total") { t.Fatalf("expected prometheus metrics body, got %q", res.Body.String()) } } func TestUIFallbackHandlesDeepLinks(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), ui: newUIRenderer(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/backup", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } if !strings.Contains(res.Body.String(), `
`) { t.Fatalf("expected UI index body, got %q", res.Body.String()) } } func TestUIFallbackDoesNotMaskMissingAssets(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), ui: newUIRenderer(), } srv.handler = http.HandlerFunc(srv.route) req := httptest.NewRequest(http.MethodGet, "/assets/missing.js", nil) res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusNotFound { t.Fatalf("expected 404, got %d: %s", res.Code, res.Body.String()) } } func TestPoliciesCRUD(t *testing.T) { srv := &Server{ cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn", Namespace: "maintenance", PolicySecretName: "soteria-policies"}, client: &fakeKubeClient{}, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{}, } srv.handler = http.HandlerFunc(srv.route) create := httptest.NewRequest(http.MethodPost, "/v1/policies", strings.NewReader(`{"namespace":"apps","interval_hours":6}`)) create.Header.Set("Content-Type", "application/json") createRes := httptest.NewRecorder() srv.Handler().ServeHTTP(createRes, create) if createRes.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", createRes.Code, createRes.Body.String()) } var created api.BackupPolicy if err := json.Unmarshal(createRes.Body.Bytes(), &created); err != nil { t.Fatalf("decode policy: %v", err) } if created.Namespace != "apps" || created.IntervalHours != 6 { t.Fatalf("unexpected created policy: %#v", created) } listReq := httptest.NewRequest(http.MethodGet, "/v1/policies", nil) listRes := httptest.NewRecorder() srv.Handler().ServeHTTP(listRes, listReq) if listRes.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", listRes.Code, listRes.Body.String()) } var listPayload api.BackupPolicyListResponse if err := json.Unmarshal(listRes.Body.Bytes(), &listPayload); err != nil { t.Fatalf("decode list: %v", err) } if len(listPayload.Policies) != 1 { t.Fatalf("expected one policy, got %#v", listPayload.Policies) } deleteReq := httptest.NewRequest(http.MethodDelete, "/v1/policies/"+url.PathEscape(created.ID), nil) deleteRes := httptest.NewRecorder() srv.Handler().ServeHTTP(deleteRes, deleteReq) if deleteRes.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", deleteRes.Code, deleteRes.Body.String()) } } func TestNamespaceBackupDryRun(t *testing.T) { srv := &Server{ cfg: &config.Config{ AuthRequired: false, BackupDriver: "longhorn", }, client: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data-a", VolumeName: "pv-apps-a", Phase: "Bound"}, {Namespace: "apps", Name: "data-b", VolumeName: "pv-apps-b", Phase: "Bound"}, {Namespace: "infra", Name: "data-c", VolumeName: "pv-infra-c", Phase: "Bound"}, }, }, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{}, } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","dry_run":true}` req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(body)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.NamespaceBackupResponse if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil { t.Fatalf("decode payload: %v", err) } if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 { t.Fatalf("unexpected namespace backup payload: %#v", payload) } } func TestNamespaceRestoreDryRun(t *testing.T) { srv := &Server{ cfg: &config.Config{ AuthRequired: false, BackupDriver: "longhorn", }, client: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "cache", VolumeName: "pv-apps-cache", Phase: "Bound"}, {Namespace: "apps", Name: "models", VolumeName: "pv-apps-models", Phase: "Bound"}, }, }, longhorn: &fakeLonghornClient{}, metrics: newTelemetry(), policies: map[string]api.BackupPolicy{}, } srv.handler = http.HandlerFunc(srv.route) body := `{"namespace":"apps","target_namespace":"restore","target_prefix":"drill","dry_run":true}` req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(body)) req.Header.Set("Content-Type", "application/json") res := httptest.NewRecorder() srv.Handler().ServeHTTP(res, req) if res.Code != http.StatusOK { t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String()) } var payload api.NamespaceRestoreResponse if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil { t.Fatalf("decode payload: %v", err) } if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 { t.Fatalf("unexpected namespace restore payload: %#v", payload) } for _, result := range payload.Results { if !strings.HasPrefix(result.TargetPVC, "drill-") { t.Fatalf("expected target pvc with prefix drill-, got %q", result.TargetPVC) } } }