soteria/internal/server/server_test.go

package server

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/soteria/internal/api"
	"scm.bstein.dev/bstein/soteria/internal/config"
	"scm.bstein.dev/bstein/soteria/internal/k8s"
	"scm.bstein.dev/bstein/soteria/internal/longhorn"

	corev1 "k8s.io/api/core/v1"
)

type fakeKubeClient struct {
	pvcs                    []k8s.PVCSummary
	backupJobs              map[string][]k8s.BackupJobSummary
	jobLogs                 map[string]string
	backupJobCalls          int
	simulateJobListMutation bool
	lastBackupReq           api.BackupRequest
	lastRestoreReq          api.RestoreTestRequest
	targetExists            bool
	secretData              map[string][]byte
}

func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
	return namespace + "-" + pvcName + "-pv", nil, nil, nil
}

func (f *fakeKubeClient) CreateBackupJob(_ context.Context, _ *config.Config, req api.BackupRequest) (string, string, error) {
	f.backupJobCalls++
	f.lastBackupReq = req
	if f.simulateJobListMutation {
		if f.backupJobs == nil {
			f.backupJobs = map[string][]k8s.BackupJobSummary{}
		}
		key := req.Namespace + "/" + req.PVC
		f.backupJobs[key] = append([]k8s.BackupJobSummary{
			{
				Name:      fmt.Sprintf("soteria-backup-%s-%d", req.PVC, f.backupJobCalls),
				Namespace: req.Namespace,
				PVC:       req.PVC,
				CreatedAt: time.Now().UTC(),
				State:     "Pending",
			},
		}, f.backupJobs[key]...)
	}
	return "backup-job", "backup-secret", nil
}

func (f *fakeKubeClient) CreateRestoreJob(_ context.Context, _ *config.Config, req api.RestoreTestRequest) (string, string, error) {
	f.lastRestoreReq = req
	return "restore-job", "restore-secret", nil
}

func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, error) {
	return f.pvcs, nil
}

func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
	if f.backupJobs == nil {
		return nil, nil
	}
	key := namespace + "/" + pvc
	items := f.backupJobs[key]
	out := make([]k8s.BackupJobSummary, len(items))
	copy(out, items)
	return out, nil
}

func (f *fakeKubeClient) ListBackupJobs(_ context.Context, namespace string) ([]k8s.BackupJobSummary, error) {
	if f.backupJobs == nil {
		return nil, nil
	}
	out := []k8s.BackupJobSummary{}
	for key, items := range f.backupJobs {
		prefix := namespace + "/"
		if !strings.HasPrefix(key, prefix) {
			continue
		}
		for _, item := range items {
			copyItem := item
			if copyItem.Namespace == "" {
				copyItem.Namespace = namespace
			}
			if copyItem.PVC == "" {
				copyItem.PVC = strings.TrimPrefix(key, prefix)
			}
			out = append(out, copyItem)
		}
	}
	return out, nil
}

func (f *fakeKubeClient) ReadBackupJobLog(_ context.Context, namespace, jobName string) (string, error) {
	if f.jobLogs == nil {
		return "", nil
	}
	key := namespace + "/" + jobName
	return f.jobLogs[key], nil
}

func (f *fakeKubeClient) PersistentVolumeClaimExists(_ context.Context, _, _ string) (bool, error) {
	return f.targetExists, nil
}

func (f *fakeKubeClient) LoadSecretData(_ context.Context, _, _, key string) ([]byte, error) {
	if f.secretData == nil {
		return nil, nil
	}
	value, ok := f.secretData[key]
	if !ok {
		return nil, nil
	}
	copyValue := make([]byte, len(value))
	copy(copyValue, value)
	return copyValue, nil
}

func (f *fakeKubeClient) SaveSecretData(_ context.Context, _, _, key string, value []byte, _ map[string]string) error {
	if f.secretData == nil {
		f.secretData = map[string][]byte{}
	}
	copyValue := make([]byte, len(value))
	copy(copyValue, value)
	f.secretData[key] = copyValue
	return nil
}

type fakeLonghornClient struct {
	backups            []longhorn.Backup
	createSnapshotName string
	snapshotBackupName string
}

func (f *fakeLonghornClient) CreateSnapshot(_ context.Context, volume, name string, labels map[string]string) error {
	f.createSnapshotName = name
	return nil
}

func (f *fakeLonghornClient) SnapshotBackup(_ context.Context, volume, name string, labels map[string]string, backupMode string) (*longhorn.Volume, error) {
	f.snapshotBackupName = name
	return &longhorn.Volume{Name: volume}, nil
}

func (f *fakeLonghornClient) GetVolume(_ context.Context, volume string) (*longhorn.Volume, error) {
	return &longhorn.Volume{Name: volume, Size: "1073741824", NumberOfReplicas: 2}, nil
}

func (f *fakeLonghornClient) CreateVolumeFromBackup(_ context.Context, name, size string, replicas int, backupURL string) (*longhorn.Volume, error) {
	return &longhorn.Volume{Name: name, Size: size, NumberOfReplicas: replicas}, nil
}

func (f *fakeLonghornClient) CreatePVC(_ context.Context, volumeName, namespace, pvcName string) error {
	return nil
}

func (f *fakeLonghornClient) DeleteVolume(_ context.Context, volumeName string) error {
	return nil
}

func (f *fakeLonghornClient) FindBackup(_ context.Context, volumeName, snapshot string) (*longhorn.Backup, error) {
	return &longhorn.Backup{Name: "backup-latest", URL: "s3://bucket/backup-latest", State: "Completed"}, nil
}

func (f *fakeLonghornClient) ListBackups(_ context.Context, volumeName string) ([]longhorn.Backup, error) {
	return f.backups, nil
}

func TestNewInitializesCoreServerFields(t *testing.T) {
	srv := New(&config.Config{}, nil, nil)

	if srv == nil {
		t.Fatalf("expected server instance")
	}
	if srv.metrics == nil {
		t.Fatalf("expected telemetry to be initialized")
	}
	if srv.ui == nil {
		t.Fatalf("expected UI renderer to be initialized")
	}
	if srv.handler == nil {
		t.Fatalf("expected handler to be initialized")
	}
	if srv.policies == nil || srv.jobUsage == nil || srv.usageStore == nil {
		t.Fatalf("expected server maps to be initialized: %#v", srv)
	}
}

func TestHealthAndReadyEndpoints(t *testing.T) {
	srv := New(&config.Config{}, nil, nil)

	testCases := []struct {
		path   string
		status string
	}{
		{path: "/healthz", status: "ok"},
		{path: "/readyz", status: "ready"},
	}

	for _, tc := range testCases {
		req := httptest.NewRequest(http.MethodGet, tc.path, nil)
		res := httptest.NewRecorder()
		srv.Handler().ServeHTTP(res, req)

		if res.Code != http.StatusOK {
			t.Fatalf("%s expected 200, got %d: %s", tc.path, res.Code, res.Body.String())
		}

		var payload map[string]string
		if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
			t.Fatalf("%s decode response: %v", tc.path, err)
		}
		if payload["status"] != tc.status {
			t.Fatalf("%s expected status %q, got %#v", tc.path, tc.status, payload)
		}
	}
}

func TestWhoAmIReturnsForwardedIdentity(t *testing.T) {
	srv := New(&config.Config{AuthRequired: true}, nil, nil)

	req := httptest.NewRequest(http.MethodGet, "/v1/whoami", nil)
	req.Header.Set("X-Forwarded-User", "brad")
	req.Header.Set("X-Forwarded-Email", "brad@bstein.dev")
	req.Header.Set("X-Forwarded-Groups", "/ops,/dev")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.AuthInfoResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode whoami response: %v", err)
	}
	if !payload.Authenticated || payload.User != "brad" || payload.Email != "brad@bstein.dev" {
		t.Fatalf("unexpected whoami payload: %#v", payload)
	}
	if len(payload.Groups) != 2 || payload.Groups[0] != "ops" || payload.Groups[1] != "dev" {
		t.Fatalf("unexpected whoami groups: %#v", payload.Groups)
	}
}

func TestWhoAmIRejectsUnsupportedMethod(t *testing.T) {
	srv := New(&config.Config{}, nil, nil)

	req := httptest.NewRequest(http.MethodPost, "/v1/whoami", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusMethodNotAllowed {
		t.Fatalf("expected 405, got %d: %s", res.Code, res.Body.String())
	}
}

func TestRootRejectsUnsupportedMethod(t *testing.T) {
	srv := New(&config.Config{}, nil, nil)

	req := httptest.NewRequest(http.MethodPost, "/", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusMethodNotAllowed {
		t.Fatalf("expected 405, got %d: %s", res.Code, res.Body.String())
	}
}

func TestRootFailsWhenUIRendererUnavailable(t *testing.T) {
	srv := New(&config.Config{}, nil, nil)
	srv.ui = nil

	req := httptest.NewRequest(http.MethodGet, "/", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusInternalServerError {
		t.Fatalf("expected 500, got %d: %s", res.Code, res.Body.String())
	}
}

func TestStartSeedsInitialBackgroundState(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			AuthRequired:            false,
			BackupDriver:            "longhorn",
			BackupMaxAge:            24 * time.Hour,
			MetricsRefreshInterval:  time.Hour,
			PolicyEvalInterval:      time.Hour,
			B2Enabled:               false,
			Namespace:               "maintenance",
			PolicySecretName:        "soteria-policies",
			UsageSecretName:         "",
		},
		client:     &fakeKubeClient{},
		longhorn:   &fakeLonghornClient{},
		metrics:    newTelemetry(),
		ui:         newUIRenderer(),
		policies:   map[string]api.BackupPolicy{},
		jobUsage:   map[string]resticJobUsageCacheEntry{},
		usageStore: map[string]resticPersistedUsageEntry{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	ctx, cancel := context.WithCancel(context.Background())
	srv.Start(ctx)
	cancel()

	b2Usage := srv.getB2Usage()
	if b2Usage.Enabled {
		t.Fatalf("expected B2 usage to remain disabled, got %#v", b2Usage)
	}
}

func TestQueryBoolParsesTruthyValues(t *testing.T) {
	if !queryBool("true") || !queryBool(" yes ") || !queryBool("1") || !queryBool("ON") {
		t.Fatalf("expected truthy values to parse as true")
	}
	if queryBool("false") || queryBool("") || queryBool("0") {
		t.Fatalf("expected falsey values to parse as false")
	}
}

func TestHandleB2UsageRejectsUnsupportedMethod(t *testing.T) {
	srv := &Server{cfg: &config.Config{}, metrics: newTelemetry()}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodPost, "/v1/b2", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusMethodNotAllowed {
		t.Fatalf("expected 405, got %d: %s", res.Code, res.Body.String())
	}
}

func TestHandleB2UsageReturnsCachedSnapshot(t *testing.T) {
	srv := &Server{
		cfg:     &config.Config{B2Enabled: false},
		metrics: newTelemetry(),
		b2Usage: api.B2UsageResponse{Enabled: false, Endpoint: "cached-endpoint"},
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/b2", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.B2UsageResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode b2 response: %v", err)
	}
	if payload.Endpoint != "cached-endpoint" {
		t.Fatalf("expected cached snapshot, got %#v", payload)
	}
}

func TestRefreshB2UsageDisabledStoresDisabledSnapshot(t *testing.T) {
	srv := &Server{
		cfg:     &config.Config{B2Enabled: false},
		metrics: newTelemetry(),
	}

	srv.refreshB2Usage(context.Background())

	if usage := srv.getB2Usage(); usage.Enabled {
		t.Fatalf("expected disabled usage snapshot, got %#v", usage)
	}
}

func TestResolveB2CredentialsFromConfigValues(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			B2Endpoint:        "https://s3.us-west-000.backblazeb2.com",
			B2Region:          "us-west-000",
			B2AccessKeyID:     "abc",
			B2SecretAccessKey: "def",
		},
		client: &fakeKubeClient{},
	}

	creds, err := srv.resolveB2Credentials(context.Background())
	if err != nil {
		t.Fatalf("resolve b2 credentials: %v", err)
	}
	if creds.Endpoint != "https://s3.us-west-000.backblazeb2.com" || creds.Region != "us-west-000" {
		t.Fatalf("unexpected endpoint/region: %#v", creds)
	}
	if creds.AccessKeyID != "abc" || creds.SecretAccessKey != "def" {
		t.Fatalf("unexpected keys: %#v", creds)
	}
}

func TestResolveB2CredentialsLoadsSecretValues(t *testing.T) {
	kube := &fakeKubeClient{
		secretData: map[string][]byte{
			"AWS_ACCESS_KEY_ID":     []byte("abc"),
			"AWS_SECRET_ACCESS_KEY": []byte("def"),
			"AWS_ENDPOINT":          []byte("https://s3.us-west-123.backblazeb2.com"),
		},
	}
	srv := &Server{
		cfg: &config.Config{
			Namespace:         "atlas",
			B2SecretNamespace: "atlas",
			B2SecretName:      "b2-creds",
			B2AccessKeyField:  "AWS_ACCESS_KEY_ID",
			B2SecretKeyField:  "AWS_SECRET_ACCESS_KEY",
			B2EndpointField:   "AWS_ENDPOINT",
		},
		client: kube,
	}

	creds, err := srv.resolveB2Credentials(context.Background())
	if err != nil {
		t.Fatalf("resolve b2 credentials from secret: %v", err)
	}
	if creds.Region != "us-west-123" {
		t.Fatalf("expected inferred region, got %#v", creds)
	}
	if creds.AccessKeyID != "abc" || creds.SecretAccessKey != "def" {
		t.Fatalf("unexpected secret credentials: %#v", creds)
	}
}

func TestResolveB2CredentialsRejectsMissingValues(t *testing.T) {
	srv := &Server{
		cfg:    &config.Config{},
		client: &fakeKubeClient{},
	}

	if _, err := srv.resolveB2Credentials(context.Background()); err == nil || !strings.Contains(err.Error(), "B2 endpoint is not configured") {
		t.Fatalf("expected missing endpoint error, got %v", err)
	}
}

func TestLoadB2SecretValueValidatesInputs(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			B2SecretNamespace: "atlas",
			B2SecretName:      "b2-creds",
		},
		client: &fakeKubeClient{},
	}

	if _, err := srv.loadB2SecretValue(context.Background(), " "); err == nil || !strings.Contains(err.Error(), "empty") {
		t.Fatalf("expected empty key error, got %v", err)
	}
	if _, err := srv.loadB2SecretValue(context.Background(), "AWS_ACCESS_KEY_ID"); err == nil || !strings.Contains(err.Error(), "is empty") {
		t.Fatalf("expected empty secret value error, got %v", err)
	}
}

func TestNormalizeS3Endpoint(t *testing.T) {
	host, secure, err := normalizeS3Endpoint("https://s3.us-west-000.backblazeb2.com")
	if err != nil || host != "s3.us-west-000.backblazeb2.com" || !secure {
		t.Fatalf("unexpected https endpoint parse: %q %v %v", host, secure, err)
	}
	host, secure, err = normalizeS3Endpoint("http://minio.internal:9000")
	if err != nil || host != "minio.internal:9000" || secure {
		t.Fatalf("unexpected http endpoint parse: %q %v %v", host, secure, err)
	}
	host, secure, err = normalizeS3Endpoint("minio.internal:9000/")
	if err != nil || host != "minio.internal:9000" || !secure {
		t.Fatalf("unexpected bare endpoint parse: %q %v %v", host, secure, err)
	}
	if _, _, err := normalizeS3Endpoint(""); err == nil {
		t.Fatalf("expected empty endpoint error")
	}
	if _, _, err := normalizeS3Endpoint("https://"); err == nil {
		t.Fatalf("expected missing host error")
	}
}

func TestInferB2RegionFallsBackWhenNeeded(t *testing.T) {
	if region := inferB2Region("https://s3.us-west-009.backblazeb2.com", "fallback"); region != "us-west-009" {
		t.Fatalf("expected inferred region, got %q", region)
	}
	if region := inferB2Region("http://minio.internal:9000", "fallback"); region != "fallback" {
		t.Fatalf("expected fallback region, got %q", region)
	}
}

func TestProtectedInventoryRequiresAuth(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn"},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusUnauthorized {
		t.Fatalf("expected 401, got %d", res.Code)
	}
}

func TestProtectedInventoryAllowsMaintenanceGroup(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour},
		client:   &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}},
		longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	req.Header.Set("X-Auth-Request-User", "brad")
	req.Header.Set("X-Auth-Request-Groups", "/maintenance")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.InventoryResponse
	if err := json.NewDecoder(strings.NewReader(res.Body.String())).Decode(&payload); err != nil {
		t.Fatalf("decode inventory: %v", err)
	}
	if len(payload.Namespaces) != 1 || payload.Namespaces[0].Name != "apps" {
		t.Fatalf("unexpected inventory payload: %#v", payload)
	}
}

func TestProtectedInventoryAllowsForwardedHeaders(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: true, AllowedGroups: []string{"admin", "maintenance"}, BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour},
		client:   &fakeKubeClient{pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"}}},
		longhorn: &fakeLonghornClient{backups: []longhorn.Backup{{Name: "backup-1", Created: "2026-04-12T00:00:00Z", State: "Completed", URL: "s3://bucket/backup-1"}}},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	req.Header.Set("X-Forwarded-User", "brad")
	req.Header.Set("X-Forwarded-Groups", "/ops;/maintenance")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
}

func TestRestoreRejectsExistingTargetPVC(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
		client:   &fakeKubeClient{targetExists: true},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"restore-data"}`
	req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusConflict {
		t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String())
	}
}

func TestRestoreRejectsInvalidTargetNamespace(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","target_namespace":"Apps_Ns","target_pvc":"restore-data","backup_url":"s3://bucket/backup"}`
	req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusBadRequest {
		t.Fatalf("expected 400, got %d: %s", res.Code, res.Body.String())
	}
}

func TestRestoreRejectsSameSourceAndTarget(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false, BackupDriver: "longhorn"},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","target_namespace":"apps","target_pvc":"data","backup_url":"s3://bucket/backup"}`
	req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusConflict {
		t.Fatalf("expected 409, got %d: %s", res.Code, res.Body.String())
	}
}

func TestBackupCreatesSnapshotBeforeBackup(t *testing.T) {
	lh := &fakeLonghornClient{}
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false, BackupDriver: "longhorn", LonghornBackupMode: "incremental"},
		client:   &fakeKubeClient{},
		longhorn: lh,
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","dry_run":false}`
	req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(body))
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	if lh.createSnapshotName == "" {
		t.Fatalf("expected snapshot creation call")
	}
	if lh.snapshotBackupName == "" {
		t.Fatalf("expected snapshot backup call")
	}
	if lh.createSnapshotName != lh.snapshotBackupName {
		t.Fatalf("expected same snapshot and backup name, got snapshot=%q backup=%q", lh.createSnapshotName, lh.snapshotBackupName)
	}
	if !strings.HasPrefix(lh.createSnapshotName, "soteria-backup-apps-data-") {
		t.Fatalf("unexpected generated backup name %q", lh.createSnapshotName)
	}
}

func TestResticBackupDefaultsDedupeEnabled(t *testing.T) {
	kube := &fakeKubeClient{}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","dry_run":false}`
	req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	if kube.lastBackupReq.Dedupe == nil || !*kube.lastBackupReq.Dedupe {
		t.Fatalf("expected dedupe default true, got %#v", kube.lastBackupReq.Dedupe)
	}
	if kube.lastBackupReq.KeepLast == nil || *kube.lastBackupReq.KeepLast != 0 {
		t.Fatalf("expected keep_last default 0, got %#v", kube.lastBackupReq.KeepLast)
	}

	var payload api.BackupResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode response: %v", err)
	}
	if !payload.Dedupe {
		t.Fatalf("expected response dedupe=true, got %#v", payload)
	}
	if payload.KeepLast != 0 {
		t.Fatalf("expected response keep_last=0, got %#v", payload)
	}
}

func TestResticInventoryUsesCompletedBackupJobs(t *testing.T) {
	completedAt := time.Now().UTC().Add(-2 * time.Hour)
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
			},
			backupJobs: map[string][]k8s.BackupJobSummary{
				"apps/data": {
					{
						Name:           "soteria-backup-data-20260413-000000",
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      completedAt.Add(-2 * time.Minute),
						CompletionTime: completedAt,
						State:          "Completed",
					},
				},
			},
			jobLogs: map[string]string{
				"apps/soteria-backup-data-20260413-000000": "Added to the repository: 25.000 MiB (10.500 MiB stored)",
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		jobUsage: map[string]resticJobUsageCacheEntry{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.InventoryResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode inventory: %v", err)
	}
	if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 {
		t.Fatalf("unexpected inventory payload: %#v", payload)
	}
	entry := payload.Namespaces[0].PVCs[0]
	if !entry.Healthy {
		t.Fatalf("expected healthy entry, got %#v", entry)
	}
	if entry.HealthReason != "fresh" {
		t.Fatalf("expected fresh health reason, got %#v", entry.HealthReason)
	}
	if entry.CompletedBackups != 1 || entry.BackupCount != 1 {
		t.Fatalf("expected one completed backup, got %#v", entry)
	}
	if entry.LastBackupSizeBytes <= 0 {
		t.Fatalf("expected restic stored bytes from job logs, got %#v", entry.LastBackupSizeBytes)
	}
	if entry.TotalBackupSizeBytes <= 0 {
		t.Fatalf("expected restic total stored bytes from job logs, got %#v", entry.TotalBackupSizeBytes)
	}
}

func TestResticInventoryUsesPersistedUsageWhenLogsGone(t *testing.T) {
	completedAt := time.Now().UTC().Add(-90 * time.Minute)
	jobName := "soteria-backup-data-20260413-010000"
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
			},
			backupJobs: map[string][]k8s.BackupJobSummary{
				"apps/data": {
					{
						Name:           jobName,
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      completedAt.Add(-2 * time.Minute),
						CompletionTime: completedAt,
						State:          "Completed",
					},
				},
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		jobUsage: map[string]resticJobUsageCacheEntry{},
		usageStore: map[string]resticPersistedUsageEntry{
			"apps/" + jobName: {
				Bytes:     4096,
				UpdatedAt: time.Now().UTC().Format(time.RFC3339),
			},
		},
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.InventoryResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode inventory: %v", err)
	}
	entry := payload.Namespaces[0].PVCs[0]
	if entry.LastBackupSizeBytes != 4096 {
		t.Fatalf("expected persisted last backup bytes, got %#v", entry.LastBackupSizeBytes)
	}
	if entry.TotalBackupSizeBytes != 4096 {
		t.Fatalf("expected persisted total backup bytes, got %#v", entry.TotalBackupSizeBytes)
	}
}

func TestResticInventoryKeepLastLimitsTotalStoredSample(t *testing.T) {
	now := time.Now().UTC()
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
			},
			backupJobs: map[string][]k8s.BackupJobSummary{
				"apps/data": {
					{
						Name:           "soteria-backup-data-20260413-030000",
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      now.Add(-3 * time.Hour),
						CompletionTime: now.Add(-3 * time.Hour),
						State:          "Completed",
						KeepLast:       1,
					},
					{
						Name:           "soteria-backup-data-20260413-020000",
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      now.Add(-4 * time.Hour),
						CompletionTime: now.Add(-4 * time.Hour),
						State:          "Completed",
						KeepLast:       1,
					},
					{
						Name:           "soteria-backup-data-20260413-010000",
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      now.Add(-5 * time.Hour),
						CompletionTime: now.Add(-5 * time.Hour),
						State:          "Completed",
						KeepLast:       1,
					},
				},
			},
			jobLogs: map[string]string{
				"apps/soteria-backup-data-20260413-030000": `{"data_added": 1200}`,
				"apps/soteria-backup-data-20260413-020000": `{"data_added": 800}`,
				"apps/soteria-backup-data-20260413-010000": `{"data_added": 600}`,
			},
		},
		longhorn:   &fakeLonghornClient{},
		metrics:    newTelemetry(),
		jobUsage:   map[string]resticJobUsageCacheEntry{},
		usageStore: map[string]resticPersistedUsageEntry{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.InventoryResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode inventory: %v", err)
	}
	entry := payload.Namespaces[0].PVCs[0]
	if entry.LastBackupSizeBytes != 1200 {
		t.Fatalf("expected latest backup size to be sampled, got %#v", entry.LastBackupSizeBytes)
	}
	if entry.TotalBackupSizeBytes != 1200 {
		t.Fatalf("expected keep_last=1 total to reflect retained snapshot only, got %#v", entry.TotalBackupSizeBytes)
	}
}

func TestResticInventoryMarksInProgressWhenOnlyActiveJobsExist(t *testing.T) {
	startedAt := time.Now().UTC().Add(-5 * time.Minute)
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
			},
			backupJobs: map[string][]k8s.BackupJobSummary{
				"apps/data": {
					{
						Name:      "soteria-backup-data-20260413-010000",
						Namespace: "apps",
						PVC:       "data",
						CreatedAt: startedAt,
						State:     "Running",
					},
				},
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/inventory", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.InventoryResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode inventory: %v", err)
	}
	if len(payload.Namespaces) != 1 || len(payload.Namespaces[0].PVCs) != 1 {
		t.Fatalf("unexpected inventory payload: %#v", payload)
	}
	entry := payload.Namespaces[0].PVCs[0]
	if entry.Healthy {
		t.Fatalf("expected in-progress backup to be unhealthy until completion, got %#v", entry)
	}
	if entry.HealthReason != "in_progress" {
		t.Fatalf("expected in_progress health reason, got %#v", entry.HealthReason)
	}
	if entry.ActiveBackups != 1 || entry.CompletedBackups != 0 {
		t.Fatalf("expected one active backup and zero completed backups, got %#v", entry)
	}
	if entry.LastJobName == "" || entry.LastJobState != "Running" {
		t.Fatalf("expected latest job metadata, got %#v", entry)
	}
	if entry.LastJobProgressPct != 70 {
		t.Fatalf("expected running progress to be 70, got %#v", entry.LastJobProgressPct)
	}
}

func TestResticBackupsEndpointReturnsLatestSelector(t *testing.T) {
	completedAt := time.Now().UTC().Add(-30 * time.Minute)
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
		},
		client: &fakeKubeClient{
			backupJobs: map[string][]k8s.BackupJobSummary{
				"apps/data": {
					{
						Name:           "soteria-backup-data-20260413-010000",
						Namespace:      "apps",
						PVC:            "data",
						CreatedAt:      completedAt.Add(-2 * time.Minute),
						CompletionTime: completedAt,
						State:          "Completed",
					},
				},
			},
			jobLogs: map[string]string{
				"apps/soteria-backup-data-20260413-010000": `{"data_added": 1200}`,
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		jobUsage: map[string]resticJobUsageCacheEntry{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/v1/backups?namespace=apps&pvc=data", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.BackupListResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode backups response: %v", err)
	}
	if len(payload.Backups) != 1 {
		t.Fatalf("expected one backup record, got %#v", payload.Backups)
	}
	if payload.Backups[0].URL != "latest" || !payload.Backups[0].Latest {
		t.Fatalf("expected latest restic selector, got %#v", payload.Backups[0])
	}
	if payload.Backups[0].Size == "" {
		t.Fatalf("expected restic backup size metadata, got %#v", payload.Backups[0])
	}
}

func TestResticRestoreUsesRepositorySelector(t *testing.T) {
	repository := "s3:https://example.invalid/atlas-soteria/isolated/apps/data"
	kube := &fakeKubeClient{}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := fmt.Sprintf(
		`{"namespace":"apps","pvc":"data","backup_url":"%s","target_namespace":"apps","target_pvc":"restore-data","dry_run":false}`,
		encodeResticSelector(repository),
	)
	req := httptest.NewRequest(http.MethodPost, "/v1/restores", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	if kube.lastRestoreReq.Repository != repository {
		t.Fatalf("expected repository selector %q, got %#v", repository, kube.lastRestoreReq.Repository)
	}
	if kube.lastRestoreReq.Snapshot != "latest" {
		t.Fatalf("expected latest snapshot selector, got %#v", kube.lastRestoreReq.Snapshot)
	}
}

func TestParseResticStoredBytesFromTextSummary(t *testing.T) {
	logBody := `
Files:           100 new,    10 changed,  900 unmodified
Added to the repository: 120.500 MiB (35.250 MiB stored)
snapshot 12345678 saved
`
	value, ok := parseResticStoredBytes(logBody)
	if !ok {
		t.Fatalf("expected to parse stored bytes from text summary")
	}
	if value <= 0 {
		t.Fatalf("expected positive parsed value, got %f", value)
	}
}

func TestParseResticStoredBytesFromJSONSummary(t *testing.T) {
	logBody := `{"message_type":"summary","files_new":1,"data_added":2048}`
	value, ok := parseResticStoredBytes(logBody)
	if !ok {
		t.Fatalf("expected to parse stored bytes from json summary")
	}
	if value != 2048 {
		t.Fatalf("expected 2048 bytes, got %f", value)
	}
}

func TestMetricsStayPublic(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: true, AllowedGroups: []string{"admin"}},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d", res.Code)
	}
	if !strings.Contains(res.Body.String(), "soteria_backup_requests_total") {
		t.Fatalf("expected prometheus metrics body, got %q", res.Body.String())
	}
}

func TestUIFallbackHandlesDeepLinks(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		ui:       newUIRenderer(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/backup", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	if !strings.Contains(res.Body.String(), `<div id="root"></div>`) {
		t.Fatalf("expected UI index body, got %q", res.Body.String())
	}
}

func TestUIFallbackDoesNotMaskMissingAssets(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		ui:       newUIRenderer(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	req := httptest.NewRequest(http.MethodGet, "/assets/missing.js", nil)
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusNotFound {
		t.Fatalf("expected 404, got %d: %s", res.Code, res.Body.String())
	}
}

func TestPoliciesCRUD(t *testing.T) {
	srv := &Server{
		cfg:      &config.Config{AuthRequired: false, BackupDriver: "longhorn", Namespace: "maintenance", PolicySecretName: "soteria-policies"},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	create := httptest.NewRequest(http.MethodPost, "/v1/policies", strings.NewReader(`{"namespace":"apps","interval_hours":6}`))
	create.Header.Set("Content-Type", "application/json")
	createRes := httptest.NewRecorder()
	srv.Handler().ServeHTTP(createRes, create)
	if createRes.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", createRes.Code, createRes.Body.String())
	}

	var created api.BackupPolicy
	if err := json.Unmarshal(createRes.Body.Bytes(), &created); err != nil {
		t.Fatalf("decode policy: %v", err)
	}
	if created.Namespace != "apps" || created.IntervalHours != 6 {
		t.Fatalf("unexpected created policy: %#v", created)
	}
	if !created.Dedupe {
		t.Fatalf("expected policy dedupe default true, got %#v", created)
	}
	if created.KeepLast != 0 {
		t.Fatalf("expected policy keep_last default 0, got %#v", created)
	}

	listReq := httptest.NewRequest(http.MethodGet, "/v1/policies", nil)
	listRes := httptest.NewRecorder()
	srv.Handler().ServeHTTP(listRes, listReq)
	if listRes.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", listRes.Code, listRes.Body.String())
	}
	var listPayload api.BackupPolicyListResponse
	if err := json.Unmarshal(listRes.Body.Bytes(), &listPayload); err != nil {
		t.Fatalf("decode list: %v", err)
	}
	if len(listPayload.Policies) != 1 {
		t.Fatalf("expected one policy, got %#v", listPayload.Policies)
	}

	deleteReq := httptest.NewRequest(http.MethodDelete, "/v1/policies/"+url.PathEscape(created.ID), nil)
	deleteRes := httptest.NewRecorder()
	srv.Handler().ServeHTTP(deleteRes, deleteReq)
	if deleteRes.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", deleteRes.Code, deleteRes.Body.String())
	}
}

func TestLoadPoliciesDefaultsDedupeEnabledWhenMissing(t *testing.T) {
	kube := &fakeKubeClient{
		secretData: map[string][]byte{
			policySecretKey: []byte(`{"policies":[{"namespace":"apps","pvc":"data","interval_hours":12,"enabled":true}]}`),
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired:     false,
			BackupDriver:     "restic",
			Namespace:        "maintenance",
			PolicySecretName: "soteria-policies",
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{},
	}
	if err := srv.loadPolicies(context.Background()); err != nil {
		t.Fatalf("load policies: %v", err)
	}
	policies := srv.listPolicies()
	if len(policies) != 1 {
		t.Fatalf("expected one policy, got %#v", policies)
	}
	if !policies[0].Dedupe {
		t.Fatalf("expected dedupe to default true for legacy policy, got %#v", policies[0])
	}
	if policies[0].KeepLast != 0 {
		t.Fatalf("expected keep_last to default 0 for legacy policy, got %#v", policies[0])
	}
}

func TestRunPolicyCycleSkipsWhenBackupAlreadyInProgress(t *testing.T) {
	startedAt := time.Now().UTC().Add(-2 * time.Hour)
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
		},
		backupJobs: map[string][]k8s.BackupJobSummary{
			"apps/data": {
				{
					Name:      "soteria-backup-data-20260413-010000",
					Namespace: "apps",
					PVC:       "data",
					CreatedAt: startedAt,
					State:     "Running",
				},
			},
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__data": {
				ID:            "apps__data",
				Namespace:     "apps",
				PVC:           "data",
				IntervalHours: 1,
				Enabled:       true,
			},
		},
	}

	srv.runPolicyCycle(context.Background())

	if kube.backupJobCalls != 0 {
		t.Fatalf("expected policy scheduler to skip while backup is in progress, got %d backup request(s)", kube.backupJobCalls)
	}
}

func TestRunPolicyCycleUsesLastAttemptTimestampWhenNoCompletedBackup(t *testing.T) {
	startedAt := time.Now().UTC().Add(-30 * time.Minute)
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
		},
		backupJobs: map[string][]k8s.BackupJobSummary{
			"apps/data": {
				{
					Name:           "soteria-backup-data-20260413-010000",
					Namespace:      "apps",
					PVC:            "data",
					CreatedAt:      startedAt,
					CompletionTime: startedAt.Add(5 * time.Minute),
					State:          "Failed",
				},
			},
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__data": {
				ID:            "apps__data",
				Namespace:     "apps",
				PVC:           "data",
				IntervalHours: 1,
				Enabled:       true,
			},
		},
	}

	srv.runPolicyCycle(context.Background())

	if kube.backupJobCalls != 0 {
		t.Fatalf("expected recent failed attempt to defer policy backup, got %d backup request(s)", kube.backupJobCalls)
	}
}

func TestRunPolicyCycleRetriesAfterIntervalFromLastAttempt(t *testing.T) {
	startedAt := time.Now().UTC().Add(-2 * time.Hour)
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
		},
		backupJobs: map[string][]k8s.BackupJobSummary{
			"apps/data": {
				{
					Name:           "soteria-backup-data-20260413-010000",
					Namespace:      "apps",
					PVC:            "data",
					CreatedAt:      startedAt,
					CompletionTime: startedAt.Add(5 * time.Minute),
					State:          "Failed",
				},
			},
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__data": {
				ID:            "apps__data",
				Namespace:     "apps",
				PVC:           "data",
				IntervalHours: 1,
				Enabled:       true,
			},
		},
	}

	srv.runPolicyCycle(context.Background())

	if kube.backupJobCalls != 1 {
		t.Fatalf("expected one policy backup after interval elapsed, got %d", kube.backupJobCalls)
	}
	if kube.lastBackupReq.Namespace != "apps" || kube.lastBackupReq.PVC != "data" {
		t.Fatalf("unexpected policy backup target: %#v", kube.lastBackupReq)
	}
}

func TestRunPolicyCycleDoesNotStormAcrossRepeatedTicksWhenBackupInProgress(t *testing.T) {
	startedAt := time.Now().UTC().Add(-5 * time.Minute)
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
		},
		backupJobs: map[string][]k8s.BackupJobSummary{
			"apps/data": {
				{
					Name:      "soteria-backup-data-active",
					Namespace: "apps",
					PVC:       "data",
					CreatedAt: startedAt,
					State:     "Running",
				},
			},
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__data": {
				ID:            "apps__data",
				Namespace:     "apps",
				PVC:           "data",
				IntervalHours: 1,
				Enabled:       true,
			},
		},
	}

	for range 30 {
		srv.runPolicyCycle(context.Background())
	}

	if kube.backupJobCalls != 0 {
		t.Fatalf("expected no policy backups while active job exists across repeated ticks, got %d", kube.backupJobCalls)
	}
}

func TestRunPolicyCycleCreatesAtMostOneJobAcrossRepeatedTicksAfterDueFailure(t *testing.T) {
	startedAt := time.Now().UTC().Add(-2 * time.Hour)
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data", VolumeName: "pv-apps-data", Phase: "Bound"},
		},
		backupJobs: map[string][]k8s.BackupJobSummary{
			"apps/data": {
				{
					Name:           "soteria-backup-data-old-failed",
					Namespace:      "apps",
					PVC:            "data",
					CreatedAt:      startedAt,
					CompletionTime: startedAt.Add(5 * time.Minute),
					State:          "Failed",
				},
			},
		},
		simulateJobListMutation: true,
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
			BackupMaxAge: 24 * time.Hour,
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__data": {
				ID:            "apps__data",
				Namespace:     "apps",
				PVC:           "data",
				IntervalHours: 1,
				Enabled:       true,
			},
		},
	}

	for range 20 {
		srv.runPolicyCycle(context.Background())
	}

	if kube.backupJobCalls != 1 {
		t.Fatalf("expected exactly one policy backup after due failure and then suppression by in-progress job, got %d", kube.backupJobCalls)
	}
}

func TestNamespaceBackupDryRun(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "longhorn",
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "data-a", VolumeName: "pv-apps-a", Phase: "Bound"},
				{Namespace: "apps", Name: "data-b", VolumeName: "pv-apps-b", Phase: "Bound"},
				{Namespace: "infra", Name: "data-c", VolumeName: "pv-infra-c", Phase: "Bound"},
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","dry_run":true}`
	req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	var payload api.NamespaceBackupResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode payload: %v", err)
	}
	if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
		t.Fatalf("unexpected namespace backup payload: %#v", payload)
	}
	if payload.KeepLast != 0 {
		t.Fatalf("expected namespace backup keep_last default 0, got %#v", payload)
	}
}

func TestNamespaceBackupUsesDedupeFlag(t *testing.T) {
	kube := &fakeKubeClient{
		pvcs: []k8s.PVCSummary{
			{Namespace: "apps", Name: "data-a", VolumeName: "pv-apps-a", Phase: "Bound"},
		},
	}
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
		},
		client:   kube,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","dry_run":false,"dedupe":false,"keep_last":1}`
	req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}

	var payload api.NamespaceBackupResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode payload: %v", err)
	}
	if payload.Dedupe {
		t.Fatalf("expected response dedupe false, got %#v", payload)
	}
	if payload.KeepLast != 1 {
		t.Fatalf("expected keep_last=1, got %#v", payload)
	}
	if kube.lastBackupReq.Dedupe == nil || *kube.lastBackupReq.Dedupe {
		t.Fatalf("expected backup request dedupe false, got %#v", kube.lastBackupReq.Dedupe)
	}
	if kube.lastBackupReq.KeepLast == nil || *kube.lastBackupReq.KeepLast != 1 {
		t.Fatalf("expected backup request keep_last=1, got %#v", kube.lastBackupReq.KeepLast)
	}
}

func TestBackupRejectsNegativeKeepLast(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "restic",
		},
		client:   &fakeKubeClient{},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","pvc":"data","keep_last":-1}`
	req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusBadRequest {
		t.Fatalf("expected 400 for invalid keep_last, got %d: %s", res.Code, res.Body.String())
	}
	if !strings.Contains(res.Body.String(), "keep_last must be") {
		t.Fatalf("expected keep_last validation message, got %s", res.Body.String())
	}
}

func TestNamespaceRestoreDryRun(t *testing.T) {
	srv := &Server{
		cfg: &config.Config{
			AuthRequired: false,
			BackupDriver: "longhorn",
		},
		client: &fakeKubeClient{
			pvcs: []k8s.PVCSummary{
				{Namespace: "apps", Name: "cache", VolumeName: "pv-apps-cache", Phase: "Bound"},
				{Namespace: "apps", Name: "models", VolumeName: "pv-apps-models", Phase: "Bound"},
			},
		},
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{},
	}
	srv.handler = http.HandlerFunc(srv.route)

	body := `{"namespace":"apps","target_namespace":"restore","target_prefix":"drill","dry_run":true}`
	req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(body))
	req.Header.Set("Content-Type", "application/json")
	res := httptest.NewRecorder()
	srv.Handler().ServeHTTP(res, req)

	if res.Code != http.StatusOK {
		t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
	}
	var payload api.NamespaceRestoreResponse
	if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
		t.Fatalf("decode payload: %v", err)
	}
	if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
		t.Fatalf("unexpected namespace restore payload: %#v", payload)
	}
	for _, result := range payload.Results {
		if !strings.HasPrefix(result.TargetPVC, "drill-") {
			t.Fatalf("expected target pvc with prefix drill-, got %q", result.TargetPVC)
		}
	}
}