375 lines
14 KiB
Go
375 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"scm.bstein.dev/bstein/soteria/internal/api"
|
|
"scm.bstein.dev/bstein/soteria/internal/config"
|
|
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
|
"scm.bstein.dev/bstein/soteria/internal/longhorn"
|
|
)
|
|
|
|
type policyCycleTestKubeClient struct {
|
|
*inventoryTestKubeClient
|
|
createBackupErrForPVC map[string]error
|
|
backupRequests []api.BackupRequest
|
|
}
|
|
|
|
func (k *policyCycleTestKubeClient) CreateBackupJob(ctx context.Context, cfg *config.Config, req api.BackupRequest) (string, string, error) {
|
|
k.backupRequests = append(k.backupRequests, req)
|
|
if err := k.createBackupErrForPVC[req.PVC]; err != nil {
|
|
return "", "", err
|
|
}
|
|
return k.inventoryTestKubeClient.fakeKubeClient.CreateBackupJob(ctx, cfg, req)
|
|
}
|
|
|
|
func metricCount(samples map[string]metricSample, labels map[string]string) float64 {
|
|
sample, ok := samples[metricKey(labels)]
|
|
if !ok {
|
|
return 0
|
|
}
|
|
return sample.value
|
|
}
|
|
|
|
func findBackupRequestByPVC(items []api.BackupRequest, pvc string) (api.BackupRequest, bool) {
|
|
for _, item := range items {
|
|
if item.PVC == pvc {
|
|
return item, true
|
|
}
|
|
}
|
|
return api.BackupRequest{}, false
|
|
}
|
|
|
|
func TestRunPolicyCycleCoversInventoryErrorAndConcurrentGuard(t *testing.T) {
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{},
|
|
listPVCsErr: errors.New("inventory exploded"),
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "restic",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
PolicyEvalInterval: time.Minute,
|
|
},
|
|
client: client,
|
|
longhorn: &fakeLonghornClient{},
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 {
|
|
t.Fatalf("expected inventory_error policy metric, got %f", got)
|
|
}
|
|
if srv.running {
|
|
t.Fatalf("expected policy runner lock to be released after inventory failure")
|
|
}
|
|
|
|
srv.running = true
|
|
srv.runPolicyCycle(context.Background())
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "inventory_error"}); got != 1 {
|
|
t.Fatalf("expected concurrent guard to skip second run, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
recent := now.Add(-30 * time.Minute)
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "busy", VolumeName: "vol-busy", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "recent", VolumeName: "vol-recent", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "attempted", VolumeName: "vol-attempted", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "fail", VolumeName: "vol-fail", Phase: "Bound"},
|
|
},
|
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
|
"apps/busy": {
|
|
{Name: "job-busy", Namespace: "apps", PVC: "busy", Repository: "s3:https://repo/root/isolated/apps/busy", State: "Running", CreatedAt: recent},
|
|
},
|
|
"apps/recent": {
|
|
{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
|
|
},
|
|
"apps/attempted": {
|
|
{Name: "job-attempted", Namespace: "apps", PVC: "attempted", State: "Failed", CreatedAt: recent, KeepLast: 1},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
createBackupErrForPVC: map[string]error{
|
|
"fail": errors.New("create backup job exploded"),
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "restic",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
ResticRepository: "s3:https://repo/root",
|
|
},
|
|
client: client,
|
|
longhorn: &fakeLonghornClient{},
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true, KeepLast: 1},
|
|
"apps__data": {ID: "apps__data", Namespace: "apps", PVC: "data", IntervalHours: 1, Enabled: true, Dedupe: false, KeepLast: 2},
|
|
"apps__disabled": {ID: "apps__disabled", Namespace: "apps", PVC: "disabled", IntervalHours: 1, Enabled: false, Dedupe: false, KeepLast: 5},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if len(client.backupRequests) != 2 {
|
|
t.Fatalf("expected two executed backup requests, got %#v", client.backupRequests)
|
|
}
|
|
dataReq, ok := findBackupRequestByPVC(client.backupRequests, "data")
|
|
if !ok || dataReq.Dedupe == nil || *dataReq.Dedupe != false || dataReq.KeepLast == nil || *dataReq.KeepLast != 2 {
|
|
t.Fatalf("expected stricter PVC policy to win for data, got %#v", client.backupRequests)
|
|
}
|
|
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
|
|
t.Fatalf("expected one successful policy backup, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "backend_error"}); got != 1 {
|
|
t.Fatalf("expected one backend error policy backup, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "in_progress"}); got != 1 {
|
|
t.Fatalf("expected one in-progress policy backup, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "not_due"}); got != 2 {
|
|
t.Fatalf("expected two not-due policy backups, got %f", got)
|
|
}
|
|
|
|
if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "success"}); got != 1 {
|
|
t.Fatalf("expected one successful executed backup request, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.backupRequests, map[string]string{"driver": "restic", "result": "backend_error"}); got != 1 {
|
|
t.Fatalf("expected one failed executed backup request, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleHonorsBackupLimit(t *testing.T) {
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "alpha", VolumeName: "vol-alpha", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "beta", VolumeName: "vol-beta", Phase: "Bound"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "restic",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
ResticRepository: "s3:https://repo/root",
|
|
PolicyBackupsPerCycle: 1,
|
|
},
|
|
client: client,
|
|
longhorn: &fakeLonghornClient{},
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if len(client.backupRequests) != 1 {
|
|
t.Fatalf("expected one policy backup due to cycle limit, got %#v", client.backupRequests)
|
|
}
|
|
if client.backupRequests[0].PVC != "alpha" {
|
|
t.Fatalf("expected deterministic first backup for alpha, got %#v", client.backupRequests[0])
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "cycle_limit"}); got != 1 {
|
|
t.Fatalf("expected cycle_limit metric, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleSkipsExcludedPVCsAndBusyRepositories(t *testing.T) {
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "astreae"},
|
|
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
|
|
},
|
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
|
"apps/other": {
|
|
{Name: "job-other", Namespace: "apps", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "restic",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
ResticRepository: "s3:https://repo/root",
|
|
ExcludedStorageClasses: []string{"local-path"},
|
|
},
|
|
client: client,
|
|
longhorn: &fakeLonghornClient{},
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if len(client.backupRequests) != 0 {
|
|
t.Fatalf("expected no backup requests while shared repo is busy and cache is excluded, got %#v", client.backupRequests)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "repo_in_progress"}); got != 1 {
|
|
t.Fatalf("expected repo_in_progress metric, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "excluded"}); got != 1 {
|
|
t.Fatalf("expected excluded metric, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleSkipsLiveExclusivePVCs(t *testing.T) {
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}},
|
|
{Namespace: "apps", Name: "shared", VolumeName: "vol-shared", Phase: "Bound", AccessModes: []string{"ReadWriteMany"}},
|
|
},
|
|
pvcMounts: map[string][]k8s.PVCMount{
|
|
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
|
|
"apps/shared": {{PodName: "web-0", NodeName: "titan-07", Phase: "Running"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "restic",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
ResticRepository: "s3:https://repo/root",
|
|
},
|
|
client: client,
|
|
longhorn: &fakeLonghornClient{},
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if len(client.backupRequests) != 1 || client.backupRequests[0].PVC != "shared" {
|
|
t.Fatalf("expected only RWX pvc backup request, got %#v", client.backupRequests)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "live_rwo_mount"}); got != 1 {
|
|
t.Fatalf("expected live_rwo_mount skip metric, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
|
|
t.Fatalf("expected one successful policy backup, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleSkipsDetachedLonghornVolumesWithoutConsumingLimit(t *testing.T) {
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "detached", VolumeName: "vol-detached", Phase: "Bound"},
|
|
{Namespace: "apps", Name: "ready", VolumeName: "vol-ready", Phase: "Bound"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
longhornClient := &fakeLonghornClient{
|
|
volumes: map[string]longhorn.Volume{
|
|
"vol-detached": {Name: "vol-detached", State: "detached"},
|
|
"vol-ready": {Name: "vol-ready", State: "attached"},
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "longhorn",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
PolicyBackupsPerCycle: 1,
|
|
},
|
|
client: client,
|
|
longhorn: longhornClient,
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if longhornClient.createSnapshotName == "" || !strings.Contains(longhornClient.createSnapshotName, "apps-ready") {
|
|
t.Fatalf("expected ready volume snapshot backup after detached skip, got snapshot %q", longhornClient.createSnapshotName)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "volume_not_ready"}); got != 1 {
|
|
t.Fatalf("expected volume_not_ready metric, got %f", got)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
|
|
t.Fatalf("expected one successful policy backup, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestRunPolicyCycleThrottlesRecentLonghornAttemptAfterStaleBackup(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
stale := now.Add(-48 * time.Hour).Format(time.RFC3339)
|
|
recentAttempt := now.Add(-30 * time.Minute).Format(time.RFC3339)
|
|
client := &policyCycleTestKubeClient{
|
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
|
fakeKubeClient: &fakeKubeClient{
|
|
pvcs: []k8s.PVCSummary{
|
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound"},
|
|
},
|
|
longhornBackups: []k8s.LonghornBackupSummary{
|
|
{Name: "backup-failed", Namespace: "apps", PVC: "data", State: "Error", CreatedAt: recentAttempt},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
longhornClient := &inventoryTestLonghornClient{
|
|
fakeLonghornClient: &fakeLonghornClient{},
|
|
listBackupsByVolume: map[string][]longhorn.Backup{
|
|
"vol-data": {
|
|
{Name: "backup-stale", Created: stale, State: "Completed", Size: "10"},
|
|
},
|
|
},
|
|
}
|
|
srv := &Server{
|
|
cfg: &config.Config{
|
|
BackupDriver: "longhorn",
|
|
BackupMaxAge: 24 * time.Hour,
|
|
},
|
|
client: client,
|
|
longhorn: longhornClient,
|
|
metrics: newTelemetry(),
|
|
policies: map[string]api.BackupPolicy{
|
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 6, Enabled: true, Dedupe: true},
|
|
},
|
|
}
|
|
|
|
srv.runPolicyCycle(context.Background())
|
|
|
|
if longhornClient.createSnapshotName != "" {
|
|
t.Fatalf("expected recent failed attempt to throttle Longhorn policy backup, got snapshot %q", longhornClient.createSnapshotName)
|
|
}
|
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "not_due"}); got != 1 {
|
|
t.Fatalf("expected not_due metric for recent failed attempt, got %f", got)
|
|
}
|
|
}
|