soteria/internal/server/policy_cycle_test.go

package server

import (
	"context"
	"errors"
	"testing"
	"time"

	"scm.bstein.dev/bstein/soteria/internal/api"
	"scm.bstein.dev/bstein/soteria/internal/config"
	"scm.bstein.dev/bstein/soteria/internal/k8s"
)

type policyCycleTestKubeClient struct {
	*inventoryTestKubeClient
	createBackupErrForPVC map[string]error
	backupRequests        []api.BackupRequest
}

func (k *policyCycleTestKubeClient) CreateBackupJob(ctx context.Context, cfg *config.Config, req api.BackupRequest) (string, string, error) {
	k.backupRequests = append(k.backupRequests, req)
	if err := k.createBackupErrForPVC[req.PVC]; err != nil {
		return "", "", err
	}
	return k.inventoryTestKubeClient.fakeKubeClient.CreateBackupJob(ctx, cfg, req)
}

func metricCount(samples map[string]metricSample, labels map[string]string) float64 {
	sample, ok := samples[metricKey(labels)]
	if !ok {
		return 0
	}
	return sample.value
}

func findBackupRequestByPVC(items []api.BackupRequest, pvc string) (api.BackupRequest, bool) {
	for _, item := range items {
		if item.PVC == pvc {
			return item, true
		}
	}
	return api.BackupRequest{}, false
}

func TestRunPolicyCycleCoversInventoryErrorAndConcurrentGuard(t *testing.T) {
	client := &policyCycleTestKubeClient{
		inventoryTestKubeClient: &inventoryTestKubeClient{
			fakeKubeClient: &fakeKubeClient{},
			listPVCsErr:    errors.New("inventory exploded"),
		},
	}
	srv := &Server{
		cfg: &config.Config{
			BackupDriver:       "restic",
			BackupMaxAge:       24 * time.Hour,
			PolicyEvalInterval: time.Minute,
		},
		client:   client,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true},
		},
	}

	srv.runPolicyCycle(context.Background())
	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 {
		t.Fatalf("expected inventory_error policy metric, got %f", got)
	}
	if srv.running {
		t.Fatalf("expected policy runner lock to be released after inventory failure")
	}

	srv.running = true
	srv.runPolicyCycle(context.Background())
	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 {
		t.Fatalf("expected concurrent guard to skip second run, got %f", got)
	}
}

func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) {
	now := time.Now().UTC()
	recent := now.Add(-30 * time.Minute)
	client := &policyCycleTestKubeClient{
		inventoryTestKubeClient: &inventoryTestKubeClient{
			fakeKubeClient: &fakeKubeClient{
				pvcs: []k8s.PVCSummary{
					{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"},
					{Namespace: "apps", Name: "busy", VolumeName: "vol-busy", Phase: "Bound"},
					{Namespace: "apps", Name: "recent", VolumeName: "vol-recent", Phase: "Bound"},
					{Namespace: "apps", Name: "attempted", VolumeName: "vol-attempted", Phase: "Bound"},
					{Namespace: "apps", Name: "fail", VolumeName: "vol-fail", Phase: "Bound"},
				},
				backupJobs: map[string][]k8s.BackupJobSummary{
					"apps/busy": {
						{Name: "job-busy", Namespace: "apps", PVC: "busy", State: "Running", CreatedAt: recent},
					},
					"apps/recent": {
						{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
					},
					"apps/attempted": {
						{Name: "job-attempted", Namespace: "apps", PVC: "attempted", State: "Failed", CreatedAt: recent, KeepLast: 1},
					},
				},
			},
		},
		createBackupErrForPVC: map[string]error{
			"fail": errors.New("create backup job exploded"),
		},
	}
	srv := &Server{
		cfg: &config.Config{
			BackupDriver:     "restic",
			BackupMaxAge:     24 * time.Hour,
			ResticRepository: "s3:https://repo/root",
		},
		client:   client,
		longhorn: &fakeLonghornClient{},
		metrics:  newTelemetry(),
		policies: map[string]api.BackupPolicy{
			"apps__all":      {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true, KeepLast: 1},
			"apps__data":     {ID: "apps__data", Namespace: "apps", PVC: "data", IntervalHours: 1, Enabled: true, Dedupe: false, KeepLast: 2},
			"apps__disabled": {ID: "apps__disabled", Namespace: "apps", PVC: "disabled", IntervalHours: 1, Enabled: false, Dedupe: false, KeepLast: 5},
		},
	}

	srv.runPolicyCycle(context.Background())

	if len(client.backupRequests) != 2 {
		t.Fatalf("expected two executed backup requests, got %#v", client.backupRequests)
	}
	dataReq, ok := findBackupRequestByPVC(client.backupRequests, "data")
	if !ok || dataReq.Dedupe == nil || *dataReq.Dedupe != false || dataReq.KeepLast == nil || *dataReq.KeepLast != 2 {
		t.Fatalf("expected stricter PVC policy to win for data, got %#v", client.backupRequests)
	}

	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
		t.Fatalf("expected one successful policy backup, got %f", got)
	}
	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "backend_error"}); got != 1 {
		t.Fatalf("expected one backend error policy backup, got %f", got)
	}
	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "in_progress"}); got != 1 {
		t.Fatalf("expected one in-progress policy backup, got %f", got)
	}
	if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "not_due"}); got != 2 {
		t.Fatalf("expected two not-due policy backups, got %f", got)
	}

	if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "success"}); got != 1 {
		t.Fatalf("expected one successful executed backup request, got %f", got)
	}
	if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "backend_error"}); got != 1 {
		t.Fatalf("expected one failed executed backup request, got %f", got)
	}
}