From 18bbd814f8561ae509518f3d2f58c4b22153feea Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 16 Jul 2026 01:34:47 -0300 Subject: [PATCH] backup: prune longhorn backups by retention --- internal/longhorn/client.go | 41 +++++++++--- internal/longhorn/client_test.go | 51 +++++++++++++++ internal/server/backup_handlers.go | 6 ++ internal/server/backup_handlers_test.go | 63 +++++++++++++++++++ internal/server/longhorn_retention.go | 84 +++++++++++++++++++++++++ internal/server/server.go | 1 + internal/server/server_test.go | 13 ++++ 7 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 internal/server/longhorn_retention.go diff --git a/internal/longhorn/client.go b/internal/longhorn/client.go index b4d1fa6..f1a0074 100644 --- a/internal/longhorn/client.go +++ b/internal/longhorn/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -73,13 +74,14 @@ type BackupVolume struct { // Backup describes a Longhorn backup record. type Backup struct { - Name string `json:"name"` - SnapshotName string `json:"snapshotName"` - VolumeName string `json:"volumeName"` - Created string `json:"created"` - State string `json:"state"` - URL string `json:"url"` - Size string `json:"size"` + Name string `json:"name"` + SnapshotName string `json:"snapshotName"` + VolumeName string `json:"volumeName"` + Created string `json:"created"` + State string `json:"state"` + URL string `json:"url"` + Size string `json:"size"` + Labels map[string]string `json:"labels"` } type backupListOutput struct { @@ -106,6 +108,10 @@ type pvcCreateInput struct { PVCName string `json:"pvcName"` } +type backupInput struct { + Name string `json:"name"` +} + // CreateSnapshot requests a snapshot on the target Longhorn volume. func (c *Client) CreateSnapshot(ctx context.Context, volume, name string, labels map[string]string) error { path := fmt.Sprintf("%s/v1/volumes/%s?action=snapshotCreate", c.baseURL, url.PathEscape(volume)) @@ -195,7 +201,7 @@ func (c *Client) GetBackupVolume(ctx context.Context, volumeName string) (*Backu return &item, nil } } - return nil, fmt.Errorf("backup volume %s not found", volumeName) + return nil, &APIError{Status: http.StatusNotFound, Message: fmt.Sprintf("backup volume %s not found", volumeName)} } else { return nil, err } @@ -218,6 +224,19 @@ func (c *Client) ListBackups(ctx context.Context, volumeName string) ([]Backup, return out.Data, nil } +// DeleteBackup removes a named backup for the Longhorn backup volume. +func (c *Client) DeleteBackup(ctx context.Context, volumeName, backupName string) error { + backupVolume, err := c.GetBackupVolume(ctx, volumeName) + if err != nil { + return err + } + deleteURL, ok := backupVolume.Actions["backupDelete"] + if !ok || deleteURL == "" { + return fmt.Errorf("backup delete action missing for volume %s", volumeName) + } + return c.doJSON(ctx, http.MethodPost, deleteURL, backupInput{Name: backupName}, nil) +} + // FindBackup locates either a specific backup or the most recent completed one. func (c *Client) FindBackup(ctx context.Context, volumeName, snapshot string) (*Backup, error) { backups, err := c.ListBackups(ctx, volumeName) @@ -263,6 +282,12 @@ func (c *Client) FindBackup(ctx context.Context, volumeName, snapshot string) (* return selected, nil } +// IsNotFound reports whether an error came from a Longhorn 404 response. +func IsNotFound(err error) bool { + var apiErr *APIError + return errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound +} + func (c *Client) doJSON(ctx context.Context, method, url string, payload any, out any) error { var body io.Reader if payload != nil { diff --git a/internal/longhorn/client_test.go b/internal/longhorn/client_test.go index 389f8a1..1d153fe 100644 --- a/internal/longhorn/client_test.go +++ b/internal/longhorn/client_test.go @@ -31,6 +31,18 @@ func TestAPIErrorErrorFormatsStatusAndMessage(t *testing.T) { } } +func TestIsNotFoundDetectsLonghorn404(t *testing.T) { + if !IsNotFound(&APIError{Status: http.StatusNotFound, Message: "missing"}) { + t.Fatalf("expected API 404 to be treated as not found") + } + if IsNotFound(&APIError{Status: http.StatusBadGateway, Message: "bad gateway"}) { + t.Fatalf("expected non-404 API error to not be treated as not found") + } + if IsNotFound(errors.New("plain error")) { + t.Fatalf("expected plain error to not be treated as not found") + } +} + func TestCoreVolumeOperationsUseExpectedRequests(t *testing.T) { ctx := context.Background() type requestRecord struct { @@ -375,6 +387,32 @@ func TestListBackupsAndFindBackupSelections(t *testing.T) { } } +func TestDeleteBackupUsesAdvertisedAction(t *testing.T) { + ctx := context.Background() + serverURL := "" + var deleteBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/backupvolumes/vol-a": + _, _ = w.Write([]byte(`{"name":"vol-a","actions":{"backupDelete":"` + serverURL + `/delete"}}`)) + case r.Method == http.MethodPost && r.URL.Path == "/delete": + deleteBody = string(raw) + _, _ = w.Write([]byte(`{"name":"vol-a"}`)) + default: + t.Fatalf("unexpected request: %s %s body=%s", r.Method, r.URL.String(), string(raw)) + } + })) + defer server.Close() + serverURL = server.URL + + client := New(server.URL) + if err := client.DeleteBackup(ctx, "vol-a", "backup-old"); err != nil { + t.Fatalf("DeleteBackup: %v", err) + } + assertBodyContains(t, deleteBody, `"name":"backup-old"`) +} + func TestListBackupsAndFindBackupErrors(t *testing.T) { ctx := context.Background() @@ -391,6 +429,19 @@ func TestListBackupsAndFindBackupErrors(t *testing.T) { } }) + t.Run("missing backupDelete action", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"name":"vol-a","actions":{}}`)) + })) + defer server.Close() + + client := New(server.URL) + err := client.DeleteBackup(ctx, "vol-a", "backup-a") + if err == nil || !strings.Contains(err.Error(), "backup delete action missing") { + t.Fatalf("expected missing backupDelete action error, got %v", err) + } + }) + t.Run("backup volume lookup error", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "volume lookup failed", http.StatusBadGateway) diff --git a/internal/server/backup_handlers.go b/internal/server/backup_handlers.go index 4d1cd3c..fb34895 100644 --- a/internal/server/backup_handlers.go +++ b/internal/server/backup_handlers.go @@ -232,6 +232,9 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque if !ready { return api.BackupResponse{}, "not_ready", errors.New(reason) } + if err := s.pruneLonghornBackups(ctx, volumeName, req.Namespace, req.PVC, resolvedKeepLast); err != nil { + return api.BackupResponse{}, "backend_error", err + } labels := map[string]string{ "soteria.bstein.dev/namespace": req.Namespace, @@ -244,6 +247,9 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque if _, err := s.longhorn.SnapshotBackup(ctx, volumeName, backupID, labels, s.cfg.LonghornBackupMode); err != nil { return api.BackupResponse{}, "backend_error", err } + if err := s.pruneLonghornBackups(ctx, volumeName, req.Namespace, req.PVC, resolvedKeepLast); err != nil { + return api.BackupResponse{}, "backend_error", err + } return response, "success", nil case "restic": _, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC) diff --git a/internal/server/backup_handlers_test.go b/internal/server/backup_handlers_test.go index efb110c..f4c9e93 100644 --- a/internal/server/backup_handlers_test.go +++ b/internal/server/backup_handlers_test.go @@ -32,6 +32,15 @@ func assertErrorResponseContains(t *testing.T, res *httptest.ResponseRecorder, w } } +func stringSliceContains(items []string, want string) bool { + for _, item := range items { + if item == want { + return true + } + } + return false +} + type backupTestKubeClient struct { *restoreTestKubeClient createBackupErr error @@ -475,6 +484,60 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) { } }) + t.Run("longhorn prunes soteria backups by keep last", func(t *testing.T) { + keepLast := 2 + fakeLonghorn := &fakeLonghornClient{backups: []longhorn.Backup{ + { + Name: "backup-keep-new", + Created: "2026-04-20T10:00:00Z", + State: "Completed", + Labels: map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"}, + }, + { + Name: "backup-keep-mid", + Created: "2026-04-19T10:00:00Z", + State: "Completed", + Labels: map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"}, + }, + { + Name: "backup-delete-old", + Created: "2026-04-18T10:00:00Z", + State: "Completed", + Labels: map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"}, + }, + { + Name: "backup-delete-failed", + Created: "2026-04-21T10:00:00Z", + State: "Error", + Labels: map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"}, + }, + { + Name: "backup-foreign", + Created: "2026-04-17T10:00:00Z", + State: "Completed", + Labels: map[string]string{longhornLabelNamespace: "other", longhornLabelPVC: "data"}, + }, + }} + srv := newBackupTestServer( + &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, + &backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}}, + &backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: fakeLonghorn}}, + ) + _, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data", KeepLast: &keepLast}, "brad") + if result != "success" || err != nil { + t.Fatalf("expected longhorn backup success with pruning, got result=%q err=%v", result, err) + } + if !stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-delete-old") { + t.Fatalf("expected old Soteria backup to be deleted, got %#v", fakeLonghorn.deletedBackups) + } + if !stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-delete-failed") { + t.Fatalf("expected failed Soteria backup to be deleted, got %#v", fakeLonghorn.deletedBackups) + } + if stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-keep-new") || stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-foreign") { + t.Fatalf("expected newest and foreign backups to be retained, got %#v", fakeLonghorn.deletedBackups) + } + }) + t.Run("longhorn detached volume", func(t *testing.T) { srv := newBackupTestServer( &config.Config{AuthRequired: false, BackupDriver: "longhorn"}, diff --git a/internal/server/longhorn_retention.go b/internal/server/longhorn_retention.go new file mode 100644 index 0000000..e7a9067 --- /dev/null +++ b/internal/server/longhorn_retention.go @@ -0,0 +1,84 @@ +package server + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "scm.bstein.dev/bstein/soteria/internal/longhorn" +) + +const ( + longhornLabelNamespace = "soteria.bstein.dev/namespace" + longhornLabelPVC = "soteria.bstein.dev/pvc" +) + +func (s *Server) pruneLonghornBackups(ctx context.Context, volumeName, namespace, pvc string, keepLast int) error { + if keepLast <= 0 { + return nil + } + + backups, err := s.longhorn.ListBackups(ctx, volumeName) + if err != nil { + if longhorn.IsNotFound(err) { + return nil + } + return err + } + + completed := make([]longhorn.Backup, 0, len(backups)) + toDelete := []longhorn.Backup{} + for _, backup := range backups { + if !longhornBackupMatchesPVC(backup, namespace, pvc) { + continue + } + switch strings.ToLower(strings.TrimSpace(backup.State)) { + case "completed": + completed = append(completed, backup) + case "error", "failed": + toDelete = append(toDelete, backup) + } + } + + sort.Slice(completed, func(i, j int) bool { + left := longhornBackupCreatedAt(completed[i]) + right := longhornBackupCreatedAt(completed[j]) + if left.Equal(right) { + return completed[i].Name > completed[j].Name + } + return left.After(right) + }) + + if len(completed) > keepLast { + toDelete = append(toDelete, completed[keepLast:]...) + } + + for _, backup := range toDelete { + if strings.TrimSpace(backup.Name) == "" { + continue + } + if err := s.longhorn.DeleteBackup(ctx, volumeName, backup.Name); err != nil { + return fmt.Errorf("delete old Longhorn backup %s for %s/%s: %w", backup.Name, namespace, pvc, err) + } + } + + return nil +} + +func longhornBackupMatchesPVC(backup longhorn.Backup, namespace, pvc string) bool { + return backup.Labels[longhornLabelNamespace] == namespace && backup.Labels[longhornLabelPVC] == pvc +} + +func longhornBackupCreatedAt(backup longhorn.Backup) time.Time { + created := strings.TrimSpace(backup.Created) + if created == "" { + return time.Time{} + } + ts, err := time.Parse(time.RFC3339, created) + if err != nil { + return time.Time{} + } + return ts +} diff --git a/internal/server/server.go b/internal/server/server.go index 6fd1a1f..7f590cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -39,6 +39,7 @@ type longhornClient interface { DeleteVolume(ctx context.Context, volumeName string) error FindBackup(ctx context.Context, volumeName, snapshot string) (*longhorn.Backup, error) ListBackups(ctx context.Context, volumeName string) ([]longhorn.Backup, error) + DeleteBackup(ctx context.Context, volumeName, backupName string) error } // Server owns HTTP routing, policy state, telemetry, and the UI renderer. diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 51bbe22..7f7ca5f 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -176,6 +176,7 @@ func (f *secretErrorKubeClient) LoadSecretData(_ context.Context, _, _, _ string type fakeLonghornClient struct { backups []longhorn.Backup volumes map[string]longhorn.Volume + deletedBackups []string createSnapshotName string snapshotBackupName string } @@ -221,6 +222,18 @@ func (f *fakeLonghornClient) ListBackups(_ context.Context, volumeName string) ( return f.backups, nil } +func (f *fakeLonghornClient) DeleteBackup(_ context.Context, volumeName, backupName string) error { + f.deletedBackups = append(f.deletedBackups, volumeName+"/"+backupName) + filtered := f.backups[:0] + for _, backup := range f.backups { + if backup.Name != backupName { + filtered = append(filtered, backup) + } + } + f.backups = filtered + return nil +} + func TestNewInitializesCoreServerFields(t *testing.T) { srv := New(&config.Config{}, nil, nil)