package server import ( "context" "errors" "strings" "testing" "time" "scm.bstein.dev/bstein/soteria/internal/api" "scm.bstein.dev/bstein/soteria/internal/config" "scm.bstein.dev/bstein/soteria/internal/k8s" "scm.bstein.dev/bstein/soteria/internal/longhorn" ) type inventoryTestKubeClient struct { *fakeKubeClient listPVCsErr error listBackupJobsErr map[string]error } func (k *inventoryTestKubeClient) ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error) { if k.listPVCsErr != nil { return nil, k.listPVCsErr } return k.fakeKubeClient.ListBoundPVCs(ctx) } func (k *inventoryTestKubeClient) ListBackupJobs(ctx context.Context, namespace string) ([]k8s.BackupJobSummary, error) { if err := k.listBackupJobsErr[namespace]; err != nil { return nil, err } return k.fakeKubeClient.ListBackupJobs(ctx, namespace) } type inventoryTestLonghornClient struct { *fakeLonghornClient listBackupsByVolume map[string][]longhorn.Backup listBackupsErr map[string]error } func (l *inventoryTestLonghornClient) ListBackups(ctx context.Context, volumeName string) ([]longhorn.Backup, error) { if err := l.listBackupsErr[volumeName]; err != nil { return nil, err } if backups, ok := l.listBackupsByVolume[volumeName]; ok { return backups, nil } return l.fakeLonghornClient.ListBackups(ctx, volumeName) } func newInventoryTestServer(cfg *config.Config, client kubeClient, longhornClient longhornClient) *Server { return &Server{ cfg: cfg, client: client, longhorn: longhornClient, metrics: newTelemetry(), } } func TestBuildInventoryLonghornSortsNamespacesAndCalculatesHealth(t *testing.T) { now := time.Now().UTC() recent := now.Add(-2 * time.Hour) old := now.Add(-48 * time.Hour) client := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "zeta", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "fast", Capacity: "10Gi", AccessModes: []string{"ReadWriteOnce"}}, {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "fast", Capacity: "20Gi", AccessModes: []string{"ReadWriteOnce"}}, }, }, } longhornClient := &inventoryTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, listBackupsByVolume: map[string][]longhorn.Backup{ "vol-data": { {Name: "backup-new", Created: recent.Format(time.RFC3339), State: "Completed", Size: "2147483648"}, {Name: "backup-old", Created: old.Format(time.RFC3339), State: "Completed", Size: "1073741824"}, }, "vol-cache": {}, }, } srv := newInventoryTestServer(&config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, }, client, longhornClient) inventory, err := srv.buildInventory(context.Background()) if err != nil { t.Fatalf("build longhorn inventory: %v", err) } if len(inventory.Namespaces) != 2 { t.Fatalf("expected two namespaces, got %#v", inventory.Namespaces) } if inventory.Namespaces[0].Name != "apps" || inventory.Namespaces[1].Name != "zeta" { t.Fatalf("expected sorted namespaces, got %#v", inventory.Namespaces) } appsPVC := inventory.Namespaces[0].PVCs[0] if !appsPVC.Healthy || appsPVC.HealthReason != "fresh" { t.Fatalf("expected healthy fresh apps pvc, got %#v", appsPVC) } if appsPVC.BackupCount != 2 || appsPVC.CompletedBackups != 2 { t.Fatalf("expected completed backup counts, got %#v", appsPVC) } if appsPVC.LastBackupSizeBytes != 2147483648 || appsPVC.TotalBackupSizeBytes != 3221225472 { t.Fatalf("expected longhorn byte totals, got %#v", appsPVC) } if appsPVC.LastBackupAt == "" || appsPVC.LastBackupAgeHours <= 0 { t.Fatalf("expected latest backup timestamp, got %#v", appsPVC) } cachePVC := inventory.Namespaces[1].PVCs[0] if cachePVC.Healthy || cachePVC.HealthReason != "missing" { t.Fatalf("expected missing backup health, got %#v", cachePVC) } } func TestBuildInventoryLonghornTracksFailedBackupCRDAttempts(t *testing.T) { recent := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339) client := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "fast"}, }, longhornBackups: []k8s.LonghornBackupSummary{ { Name: "backup-failed", Namespace: "apps", PVC: "data", State: "Error", Error: "storage cap exceeded", CreatedAt: recent, }, }, }, } longhornClient := &inventoryTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, listBackupsByVolume: map[string][]longhorn.Backup{ "vol-data": {}, }, } srv := newInventoryTestServer(&config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, }, client, longhornClient) inventory, err := srv.buildInventory(context.Background()) if err != nil { t.Fatalf("build longhorn inventory: %v", err) } entry := inventory.Namespaces[0].PVCs[0] if entry.BackupCount != 1 || entry.CompletedBackups != 0 || entry.ActiveBackups != 0 { t.Fatalf("expected failed CRD attempt to count without active/completed backups, got %#v", entry) } if entry.LastJobName != "backup-failed" || entry.LastJobState != "Error" || entry.LastJobStartedAt != recent { t.Fatalf("expected latest failed CRD metadata, got %#v", entry) } if entry.LastJobProgressPct != 100 { t.Fatalf("expected failed CRD to report terminal progress, got %#v", entry.LastJobProgressPct) } if entry.Healthy || entry.HealthReason != "missing" { t.Fatalf("expected no completed backup health, got %#v", entry) } } func TestPrefetchLonghornBackupCRDsCoversDriverFilterAndErrors(t *testing.T) { resticSrv := newInventoryTestServer(&config.Config{BackupDriver: "restic"}, &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) if backups, err := resticSrv.prefetchLonghornBackupCRDs(context.Background()); err != nil || backups != nil { t.Fatalf("expected non-longhorn driver to skip CRD prefetch, backups=%#v err=%v", backups, err) } errSrv := newInventoryTestServer(&config.Config{BackupDriver: "longhorn"}, &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{longhornBackupLookupErr: errors.New("list CRDs exploded")}}, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) if _, err := errSrv.prefetchLonghornBackupCRDs(context.Background()); err == nil || !strings.Contains(err.Error(), "list CRDs exploded") { t.Fatalf("expected CRD lookup error, got %v", err) } client := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ longhornBackups: []k8s.LonghornBackupSummary{ {Name: "older", Namespace: "apps", PVC: "data", CreatedAt: "2026-04-20T10:00:00Z"}, {Name: "newer", Namespace: "apps", PVC: "data", CreatedAt: "2026-04-20T11:00:00Z"}, {Name: "ignored", Namespace: "", PVC: "data", CreatedAt: "2026-04-20T12:00:00Z"}, }, }, } srv := newInventoryTestServer(&config.Config{BackupDriver: "longhorn"}, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) backups, err := srv.prefetchLonghornBackupCRDs(context.Background()) if err != nil { t.Fatalf("prefetch Longhorn backup CRDs: %v", err) } got := backups["apps/data"] if len(got) != 2 || got[0].Name != "newer" || got[1].Name != "older" { t.Fatalf("expected grouped backups sorted newest first, got %#v", got) } } func TestBuildInventoryMarksExcludedPVCsHealthy(t *testing.T) { client := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"}, }, }, } srv := newInventoryTestServer(&config.Config{ BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, ExcludedStorageClasses: []string{"local-path"}, }, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) inventory, err := srv.buildInventory(context.Background()) if err != nil { t.Fatalf("build inventory: %v", err) } entry := inventory.Namespaces[0].PVCs[0] if !entry.Healthy || entry.HealthReason != "excluded" || !strings.Contains(entry.Error, "local-path") { t.Fatalf("expected excluded pvc to be treated as healthy exclusion, got %#v", entry) } } func TestEnrichPVCInventoryCoversLonghornAndResticBranches(t *testing.T) { now := time.Now().UTC() recent := now.Add(-1 * time.Hour) old := now.Add(-72 * time.Hour) t.Run("longhorn branches", func(t *testing.T) { srv := newInventoryTestServer(&config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, }, &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{}}, &inventoryTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, listBackupsErr: map[string]error{ "vol-error": errors.New("list backups exploded"), }, listBackupsByVolume: map[string][]longhorn.Backup{ "vol-no-completed": { {Name: "backup-pending", Created: recent.Format(time.RFC3339), State: "InProgress"}, }, "vol-bad-time": { {Name: "backup-complete", Created: "not-a-time", State: "Completed", Size: "123"}, }, "vol-stale": { {Name: "backup-old", Created: old.Format(time.RFC3339), State: "Completed", Size: "10"}, }, }, }) testCases := []struct { entry apiPVCInventory want string ok bool err string }{ {entry: apiPVCInventory{Namespace: "apps", PVC: "err", Volume: "vol-error"}, want: "lookup_failed", ok: false, err: "list backups exploded"}, {entry: apiPVCInventory{Namespace: "apps", PVC: "none", Volume: "vol-no-completed"}, want: "no_completed", ok: false}, {entry: apiPVCInventory{Namespace: "apps", PVC: "bad", Volume: "vol-bad-time"}, want: "unknown_timestamp", ok: false}, {entry: apiPVCInventory{Namespace: "apps", PVC: "stale", Volume: "vol-stale"}, want: "stale", ok: false}, } for _, tc := range testCases { entry := tc.entry.toAPI() srv.enrichPVCInventory(context.Background(), &entry, nil, nil, nil, nil) if entry.HealthReason != tc.want || entry.Healthy != tc.ok { t.Fatalf("%s/%s: expected %q healthy=%v, got %#v", entry.Namespace, entry.PVC, tc.want, tc.ok, entry) } if tc.err != "" && entry.Error != tc.err { t.Fatalf("%s/%s: expected error %q, got %#v", entry.Namespace, entry.PVC, tc.err, entry) } } }) t.Run("restic branches", func(t *testing.T) { client := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ jobLogs: map[string]string{ "apps/job-new": `{"message_type":"summary","data_added":2048}`, "apps/job-old": `{"message_type":"summary","data_added":1024}`, }, }, } srv := newInventoryTestServer(&config.Config{ Namespace: "atlas", UsageSecretName: "restic-usage", BackupDriver: "restic", BackupMaxAge: 24 * time.Hour, }, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) resticJobsByPVC := map[string][]k8s.BackupJobSummary{ "apps/data": { {Name: "job-old", Namespace: "apps", PVC: "data", State: "Completed", CreatedAt: old, CompletionTime: old, KeepLast: 1}, {Name: "job-new", Namespace: "apps", PVC: "data", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1}, }, "apps/running": { {Name: "job-running", Namespace: "apps", PVC: "running", State: "Running", CreatedAt: recent}, }, "apps/bad-time": { {Name: "job-bad", Namespace: "apps", PVC: "bad-time", State: "Completed"}, }, } for key := range resticJobsByPVC { sortBackupJobsNewestFirst(resticJobsByPVC[key]) } entry := apiPVCInventory{Namespace: "apps", PVC: "data", Volume: "vol-data"}.toAPI() srv.enrichPVCInventory(context.Background(), &entry, resticJobsByPVC, nil, nil, nil) if !entry.Healthy || entry.HealthReason != "fresh" { t.Fatalf("expected fresh restic pvc, got %#v", entry) } if entry.LastJobName != "job-new" || entry.BackupCount != 2 || entry.CompletedBackups != 2 { t.Fatalf("expected sorted restic jobs, got %#v", entry) } if entry.LastBackupSizeBytes != 2048 || entry.TotalBackupSizeBytes != 2048 { t.Fatalf("expected retained size accounting, got %#v", entry) } running := apiPVCInventory{Namespace: "apps", PVC: "running", Volume: "vol-running"}.toAPI() srv.enrichPVCInventory(context.Background(), &running, resticJobsByPVC, nil, nil, nil) if running.Healthy || running.HealthReason != "in_progress" || running.ActiveBackups != 1 { t.Fatalf("expected in-progress restic pvc, got %#v", running) } missing := apiPVCInventory{Namespace: "apps", PVC: "missing", Volume: "vol-missing"}.toAPI() srv.enrichPVCInventory(context.Background(), &missing, resticJobsByPVC, nil, nil, nil) if missing.Healthy || missing.HealthReason != "missing" { t.Fatalf("expected missing restic pvc, got %#v", missing) } badTime := apiPVCInventory{Namespace: "apps", PVC: "bad-time", Volume: "vol-bad-time"}.toAPI() srv.enrichPVCInventory(context.Background(), &badTime, resticJobsByPVC, nil, nil, nil) if badTime.Healthy || badTime.HealthReason != "unknown_timestamp" { t.Fatalf("expected unknown timestamp restic pvc, got %#v", badTime) } lookupFailed := apiPVCInventory{Namespace: "ops", PVC: "data", Volume: "vol-ops"}.toAPI() srv.enrichPVCInventory(context.Background(), &lookupFailed, resticJobsByPVC, map[string]error{"ops": errors.New("list jobs exploded")}, nil, nil) if lookupFailed.Healthy || lookupFailed.HealthReason != "lookup_failed" || lookupFailed.Error != "list jobs exploded" { t.Fatalf("expected lookup failure restic pvc, got %#v", lookupFailed) } }) } func TestRefreshTelemetryRecordsSuccessAndFailure(t *testing.T) { successClient := &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{ pvcs: []k8s.PVCSummary{ {Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"}, }, }, } successLonghorn := &inventoryTestLonghornClient{ fakeLonghornClient: &fakeLonghornClient{}, listBackupsByVolume: map[string][]longhorn.Backup{ "vol-data": { {Name: "backup-new", Created: time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339), State: "Completed", Size: "512"}, }, }, } srv := newInventoryTestServer(&config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, }, successClient, successLonghorn) srv.refreshTelemetry(context.Background()) if srv.metrics.inventoryRefreshFailure != 0 { t.Fatalf("expected successful refresh, got failure count %f", srv.metrics.inventoryRefreshFailure) } if srv.metrics.inventoryRefreshTime == 0 { t.Fatalf("expected inventory refresh timestamp to be recorded") } if len(srv.metrics.pvcBackupHealth) != 1 || len(srv.metrics.pvcBackupCount) != 1 { t.Fatalf("expected pvc metrics to be recorded, got health=%d count=%d", len(srv.metrics.pvcBackupHealth), len(srv.metrics.pvcBackupCount)) } failingSrv := newInventoryTestServer(&config.Config{ BackupDriver: "longhorn", BackupMaxAge: 24 * time.Hour, }, &inventoryTestKubeClient{ fakeKubeClient: &fakeKubeClient{}, listPVCsErr: errors.New("inventory exploded"), }, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}) failingSrv.refreshTelemetry(context.Background()) if failingSrv.metrics.inventoryRefreshFailure != 1 { t.Fatalf("expected failed refresh metric, got %f", failingSrv.metrics.inventoryRefreshFailure) } } func TestSortBackupJobsNewestFirstUsesCompletionCreatedAndNameTiebreakers(t *testing.T) { now := time.Now().UTC() items := []k8s.BackupJobSummary{ { Name: "job-a", Namespace: "apps", PVC: "data", CreatedAt: now.Add(-3 * time.Hour), CompletionTime: now.Add(-2 * time.Hour), }, { Name: "job-c", Namespace: "apps", PVC: "data", CreatedAt: now.Add(-1 * time.Hour), }, { Name: "job-b", Namespace: "apps", PVC: "data", CreatedAt: now.Add(-4 * time.Hour), CompletionTime: now.Add(-2 * time.Hour), }, } sortBackupJobsNewestFirst(items) got := []string{items[0].Name, items[1].Name, items[2].Name} want := []string{"job-c", "job-b", "job-a"} for index := range want { if got[index] != want[index] { t.Fatalf("expected sorted names %v, got %v", want, got) } } } func TestLonghornBackupInventoryHelpers(t *testing.T) { backups := []k8s.LonghornBackupSummary{ {Name: "backup-a", CreatedAt: "2026-04-20T10:00:00Z"}, {Name: "backup-c", CreatedAt: "2026-04-20T11:00:00Z"}, {Name: "backup-b", CreatedAt: "2026-04-20T10:00:00Z"}, } sortLonghornBackupsNewestFirst(backups) got := []string{backups[0].Name, backups[1].Name, backups[2].Name} want := []string{"backup-c", "backup-b", "backup-a"} for index := range want { if got[index] != want[index] { t.Fatalf("expected sorted Longhorn backups %v, got %v", want, got) } } testCases := []struct { backup k8s.LonghornBackupSummary want int }{ {backup: k8s.LonghornBackupSummary{State: "Completed"}, want: 100}, {backup: k8s.LonghornBackupSummary{State: "Failed"}, want: 100}, {backup: k8s.LonghornBackupSummary{State: "Running", Progress: 45}, want: 45}, {backup: k8s.LonghornBackupSummary{State: "Running", Progress: 150}, want: 100}, {backup: k8s.LonghornBackupSummary{State: "Running"}, want: 20}, } for _, tc := range testCases { if got := longhornBackupProgressPct(tc.backup); got != tc.want { t.Fatalf("expected progress %d for %#v, got %d", tc.want, tc.backup, got) } } } type apiPVCInventory struct { Namespace string PVC string Volume string } func (p apiPVCInventory) toAPI() api.PVCInventory { return api.PVCInventory{ Namespace: p.Namespace, PVC: p.PVC, Volume: p.Volume, } }