soteria/internal/server/server_test.go

629 lines
21 KiB
Go

package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/soteria/internal/api"
"scm.bstein.dev/bstein/soteria/internal/config"
"scm.bstein.dev/bstein/soteria/internal/k8s"
"scm.bstein.dev/bstein/soteria/internal/longhorn"
corev1 "k8s.io/api/core/v1"
)
type fakeKubeClient struct {
pvcs []k8s.PVCSummary
backupJobs map[string][]k8s.BackupJobSummary
targetExists bool
secretData map[string][]byte
}
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
return namespace + "-" + pvcName + "-pv", nil, nil, nil
}
func (f *fakeKubeClient) CreateBackupJob(_ context.Context, _ *config.Config, _ api.BackupRequest) (string, string, error) {
return "backup-job", "backup-secret", nil
}
func (f *fakeKubeClient) CreateRestoreJob(_ context.Context, _ *config.Config, _ api.RestoreTestRequest) (string, string, error) {
return "restore-job", "restore-secret", nil
}
func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, error) {
return f.pvcs, nil
}
func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
if f.backupJobs == nil {
return nil, nil
}
key := namespace + "/" + pvc
items := f.backupJobs[key]
out := make([]k8s.BackupJobSummary, len(items))
copy(out, items)
return out, nil
}
func (f *fakeKubeClient) PersistentVolumeClaimExists(_ context.Context, _, _ string) (bool, error) {
return f.targetExists, nil
}
func (f *fakeKubeClient) LoadSecretData(_ context.Context, _, _, key string) ([]byte, error) {
if f.secretData == nil {
return nil, nil
}
value, ok := f.secretData[key]
if !ok {
return nil, nil
}
copyValue := make([]byte, len(value))
copy(copyValue, value)
return copyValue, nil
}
func (f *fakeKubeClient) SaveSecretData(_ context.Context, _, _, key string, value []byte, _ map[string]string) error {
if f.secretData == nil {
f.secretData = map[string][]byte{}
}
copyValue := make([]byte, len(value))
copy(copyValue, value)
f.secretData[key] = copyValue
return nil
}
type fakeLonghornClient struct {
backups []longhorn.Backup
createSnapshotName string
snapshotBackupName string
}
func (f *fakeLonghornClient) CreateSnapshot(_ context.Context, volume, name string, labels map[string]string) error {
f.createSnapshotName = name
return nil
}
func (f *fakeLonghornClient) SnapshotBackup(_ context.Context, volume, name string, labels map[string]string, backupMode string) (*longhorn.Volume, error) {
f.snapshotBackupName = name
return &longhorn.Volume{Name: volume}, nil
}
func (f *fakeLonghornClient) GetVolume(_ context.Context, volume string) (*longhorn.Volume, error) {
return &longhorn.Volume{Name: volume, Size: "1073741824", NumberOfReplicas: 2}, nil
}
func (f *fakeLonghornClient) CreateVolumeFromBackup(_ context.Context, name, size string, replicas int, backupURL string) (*longhorn.Volume, error) {
return &longhorn.Volume{Name: name, Size: size, NumberOfReplicas: replicas}, nil
}
func (f *fakeLonghornClient) CreatePVC(_ context.Context, volumeName, namespace, pvcName string) error {
return nil
}
func (f *fakeLonghornClient) DeleteVolume(_ context.Context, volumeName string) error {
return nil
}
func (f *fakeLonghornClient) FindBackup(_ context.Context, volumeName, snapshot string) (*longhorn.Backup, error) {
return &longhorn.Backup{Name: "backup-latest", URL: "s3://bucket/backup-latest", State: "Completed"}, nil
}
func (f *fakeLonghornClient) ListBackups(_ context.Context, volumeName string) ([]longhorn.Backup, error) {
return f.backups, nil
}
func TestProtectedInventoryRequiresAuth(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn"},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusUnauthorized {
t.Fatalf("expected 401, got %d", res.Code)
}
}
func TestProtectedInventoryAllowsMaintenanceGroup(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour},
client: &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}},
longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
req.Header.Set("X-Auth-Request-User", "brad")
req.Header.Set("X-Auth-Request-Groups", "/maintenance")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.InventoryResponse
if err := json.NewDecoder(strings.NewReader(res.Body.String())).Decode(&payload); err != nil {
t.Fatalf("decode inventory: %v", err)
}
if len(payload.Namespaces) != 1 || payload.Namespaces[0].Name != "apps" {
t.Fatalf("unexpected inventory payload: %#v", payload)
}
}
func TestProtectedInventoryAllowsForwardedHeaders(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour},
client: &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}},
longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
req.Header.Set("X-Forwarded-User", "brad")
req.Header.Set("X-Forwarded-Groups", "/ops;/maintenance")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
}
func TestRestoreRejectsExistingTargetPVC(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
client: &fakeKubeClient{targetExists: true},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"restore-data"}`
req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusConflict {
t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String())
}
}
func TestRestoreRejectsInvalidTargetNamespace(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","pvc":"data","target_namespace":"Apps_Ns","target_pvc":"restore-data","backup_url":"s3://bucket/backup"}`
req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d: %s", res.Code, res.Body.String())
}
}
func TestRestoreRejectsSameSourceAndTarget(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"data","backup_url":"s3://bucket/backup"}`
req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusConflict {
t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String())
}
}
func TestBackupCreatesSnapshotBeforeBackup(t *testing.T) {
lh := &fakeLonghornClient{}
srv := &Server{
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn", LonghornBackupMode: "incremental"},
client: &fakeKubeClient{},
longhorn: lh,
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","pvc":"data","dry_run":false}`
req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(body))
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
if lh.createSnapshotName == "" {
t.Fatalf("expected snapshot creation call")
}
if lh.snapshotBackupName == "" {
t.Fatalf("expected snapshot backup call")
}
if lh.createSnapshotName != lh.snapshotBackupName {
t.Fatalf("expected same snapshot and backup name, got snapshot=%q backup=%q", lh.createSnapshotName, lh.snapshotBackupName)
}
if !strings.HasPrefix(lh.createSnapshotName, "soteria-backup-apps-data-") {
t.Fatalf("unexpected generated backup name %q", lh.createSnapshotName)
}
}
func TestResticInventoryUsesCompletedBackupJobs(t *testing.T) {
completedAt := time.Now().UTC().Add(-2 * time.Hour)
srv := &Server{
cfg: &config.Config{
AuthRequired: false,
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
},
client: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/data": {
{
Name: "soteria-backup-data-20260413-000000",
Namespace: "apps",
PVC: "data",
CreatedAt: completedAt.Add(-2 * time.Minute),
CompletionTime: completedAt,
State: "Completed",
},
},
},
},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.InventoryResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode inventory: %v", err)
}
if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 {
t.Fatalf("unexpected inventory payload: %#v", payload)
}
entry := payload.Namespaces[0].PVCs[0]
if !entry.Healthy {
t.Fatalf("expected healthy entry, got %#v", entry)
}
if entry.HealthReason != "fresh" {
t.Fatalf("expected fresh health reason, got %#v", entry.HealthReason)
}
if entry.CompletedBackups != 1 || entry.BackupCount != 1 {
t.Fatalf("expected one completed backup, got %#v", entry)
}
}
func TestResticInventoryMarksInProgressWhenOnlyActiveJobsExist(t *testing.T) {
startedAt := time.Now().UTC().Add(-5 * time.Minute)
srv := &Server{
cfg: &config.Config{
AuthRequired: false,
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
},
client: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/data": {
{
Name: "soteria-backup-data-20260413-010000",
Namespace: "apps",
PVC: "data",
CreatedAt: startedAt,
State: "Running",
},
},
},
},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.InventoryResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode inventory: %v", err)
}
if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 {
t.Fatalf("unexpected inventory payload: %#v", payload)
}
entry := payload.Namespaces[0].PVCs[0]
if entry.Healthy {
t.Fatalf("expected in-progress backup to be unhealthy until completion, got %#v", entry)
}
if entry.HealthReason != "in_progress" {
t.Fatalf("expected in_progress health reason, got %#v", entry.HealthReason)
}
if entry.ActiveBackups != 1 || entry.CompletedBackups != 0 {
t.Fatalf("expected one active backup and zero completed backups, got %#v", entry)
}
if entry.LastJobName == "" || entry.LastJobState != "Running" {
t.Fatalf("expected latest job metadata, got %#v", entry)
}
if entry.LastJobProgressPct != 70 {
t.Fatalf("expected running progress to be 70, got %#v", entry.LastJobProgressPct)
}
}
func TestResticBackupsEndpointReturnsLatestSelector(t *testing.T) {
completedAt := time.Now().UTC().Add(-30 * time.Minute)
srv := &Server{
cfg: &config.Config{
AuthRequired: false,
BackupDriver: "restic",
},
client: &fakeKubeClient{
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/data": {
{
Name: "soteria-backup-data-20260413-010000",
Namespace: "apps",
PVC: "data",
CreatedAt: completedAt.Add(-2 * time.Minute),
CompletionTime: completedAt,
State: "Completed",
},
},
},
},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/v1/backups?namespace=apps&pvc=data", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.BackupListResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode backups response: %v", err)
}
if len(payload.Backups) != 1 {
t.Fatalf("expected one backup record, got %#v", payload.Backups)
}
if payload.Backups[0].URL != "latest" || !payload.Backups[0].Latest {
t.Fatalf("expected latest restic selector, got %#v", payload.Backups[0])
}
}
func TestMetricsStayPublic(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: true, AllowedGroups: []string{"admin"}},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", res.Code)
}
if !strings.Contains(res.Body.String(), "soteria_backup_requests_total") {
t.Fatalf("expected prometheus metrics body, got %q", res.Body.String())
}
}
func TestUIFallbackHandlesDeepLinks(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
ui: newUIRenderer(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/backup", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
if !strings.Contains(res.Body.String(), `<div id="root"></div>`) {
t.Fatalf("expected UI index body, got %q", res.Body.String())
}
}
func TestUIFallbackDoesNotMaskMissingAssets(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
ui: newUIRenderer(),
}
srv.handler = http.HandlerFunc(srv.route)
req := httptest.NewRequest(http.MethodGet, "/assets/missing.js", nil)
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusNotFound {
t.Fatalf("expected 404, got %d: %s", res.Code, res.Body.String())
}
}
func TestPoliciesCRUD(t *testing.T) {
srv := &Server{
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn", Namespace: "maintenance", PolicySecretName: "soteria-policies"},
client: &fakeKubeClient{},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{},
}
srv.handler = http.HandlerFunc(srv.route)
create := httptest.NewRequest(http.MethodPost, "/v1/policies", strings.NewReader(`{"namespace":"apps","interval_hours":6}`))
create.Header.Set("Content-Type", "application/json")
createRes := httptest.NewRecorder()
srv.Handler().ServeHTTP(createRes, create)
if createRes.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", createRes.Code, createRes.Body.String())
}
var created api.BackupPolicy
if err := json.Unmarshal(createRes.Body.Bytes(), &created); err != nil {
t.Fatalf("decode policy: %v", err)
}
if created.Namespace != "apps" || created.IntervalHours != 6 {
t.Fatalf("unexpected created policy: %#v", created)
}
listReq := httptest.NewRequest(http.MethodGet, "/v1/policies", nil)
listRes := httptest.NewRecorder()
srv.Handler().ServeHTTP(listRes, listReq)
if listRes.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", listRes.Code, listRes.Body.String())
}
var listPayload api.BackupPolicyListResponse
if err := json.Unmarshal(listRes.Body.Bytes(), &listPayload); err != nil {
t.Fatalf("decode list: %v", err)
}
if len(listPayload.Policies) != 1 {
t.Fatalf("expected one policy, got %#v", listPayload.Policies)
}
deleteReq := httptest.NewRequest(http.MethodDelete, "/v1/policies/"+url.PathEscape(created.ID), nil)
deleteRes := httptest.NewRecorder()
srv.Handler().ServeHTTP(deleteRes, deleteReq)
if deleteRes.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", deleteRes.Code, deleteRes.Body.String())
}
}
func TestNamespaceBackupDryRun(t *testing.T) {
srv := &Server{
cfg: &config.Config{
AuthRequired: false,
BackupDriver: "longhorn",
},
client: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data-a", VolumeName: "pv-apps-a", Phase: "Bound"},
{Namespace: "apps", Name: "data-b", VolumeName: "pv-apps-b", Phase: "Bound"},
{Namespace: "infra", Name: "data-c", VolumeName: "pv-infra-c", Phase: "Bound"},
},
},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{},
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","dry_run":true}`
req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.NamespaceBackupResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode payload: %v", err)
}
if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
t.Fatalf("unexpected namespace backup payload: %#v", payload)
}
}
func TestNamespaceRestoreDryRun(t *testing.T) {
srv := &Server{
cfg: &config.Config{
AuthRequired: false,
BackupDriver: "longhorn",
},
client: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "cache", VolumeName: "pv-apps-cache", Phase: "Bound"},
{Namespace: "apps", Name: "models", VolumeName: "pv-apps-models", Phase: "Bound"},
},
},
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{},
}
srv.handler = http.HandlerFunc(srv.route)
body := `{"namespace":"apps","target_namespace":"restore","target_prefix":"drill","dry_run":true}`
req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.NamespaceRestoreResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode payload: %v", err)
}
if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
t.Fatalf("unexpected namespace restore payload: %#v", payload)
}
for _, result := range payload.Results {
if !strings.HasPrefix(result.TargetPVC, "drill-") {
t.Fatalf("expected target pvc with prefix drill-, got %q", result.TargetPVC)
}
}
}