backup: prune longhorn backups by retention

internal/longhorn/client.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -80,6 +81,7 @@ type Backup struct {
 	State        string            `json:"state"`
 	URL          string            `json:"url"`
 	Size         string            `json:"size"`
+	Labels       map[string]string `json:"labels"`
 }
 
 type backupListOutput struct {
@@ -106,6 +108,10 @@ type pvcCreateInput struct {
 	PVCName   string `json:"pvcName"`
 }
 
+type backupInput struct {
+	Name string `json:"name"`
+}
+
 // CreateSnapshot requests a snapshot on the target Longhorn volume.
 func (c *Client) CreateSnapshot(ctx context.Context, volume, name string, labels map[string]string) error {
 	path := fmt.Sprintf("%s/v1/volumes/%s?action=snapshotCreate", c.baseURL, url.PathEscape(volume))
@@ -195,7 +201,7 @@ func (c *Client) GetBackupVolume(ctx context.Context, volumeName string) (*Backu
 				return &item, nil
 			}
 		}
-		return nil, fmt.Errorf("backup volume %s not found", volumeName)
+		return nil, &APIError{Status: http.StatusNotFound, Message: fmt.Sprintf("backup volume %s not found", volumeName)}
 	} else {
 		return nil, err
 	}
@@ -218,6 +224,19 @@ func (c *Client) ListBackups(ctx context.Context, volumeName string) ([]Backup,
 	return out.Data, nil
 }
 
+// DeleteBackup removes a named backup for the Longhorn backup volume.
+func (c *Client) DeleteBackup(ctx context.Context, volumeName, backupName string) error {
+	backupVolume, err := c.GetBackupVolume(ctx, volumeName)
+	if err != nil {
+		return err
+	}
+	deleteURL, ok := backupVolume.Actions["backupDelete"]
+	if !ok || deleteURL == "" {
+		return fmt.Errorf("backup delete action missing for volume %s", volumeName)
+	}
+	return c.doJSON(ctx, http.MethodPost, deleteURL, backupInput{Name: backupName}, nil)
+}
+
 // FindBackup locates either a specific backup or the most recent completed one.
 func (c *Client) FindBackup(ctx context.Context, volumeName, snapshot string) (*Backup, error) {
 	backups, err := c.ListBackups(ctx, volumeName)
@@ -263,6 +282,12 @@ func (c *Client) FindBackup(ctx context.Context, volumeName, snapshot string) (*
 	return selected, nil
 }
 
+// IsNotFound reports whether an error came from a Longhorn 404 response.
+func IsNotFound(err error) bool {
+	var apiErr *APIError
+	return errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound
+}
+
 func (c *Client) doJSON(ctx context.Context, method, url string, payload any, out any) error {
 	var body io.Reader
 	if payload != nil {

internal/longhorn/client_test.go

@@ -31,6 +31,18 @@ func TestAPIErrorErrorFormatsStatusAndMessage(t *testing.T) {
 	}
 }
 
+func TestIsNotFoundDetectsLonghorn404(t *testing.T) {
+	if !IsNotFound(&APIError{Status: http.StatusNotFound, Message: "missing"}) {
+		t.Fatalf("expected API 404 to be treated as not found")
+	}
+	if IsNotFound(&APIError{Status: http.StatusBadGateway, Message: "bad gateway"}) {
+		t.Fatalf("expected non-404 API error to not be treated as not found")
+	}
+	if IsNotFound(errors.New("plain error")) {
+		t.Fatalf("expected plain error to not be treated as not found")
+	}
+}
+
 func TestCoreVolumeOperationsUseExpectedRequests(t *testing.T) {
 	ctx := context.Background()
 	type requestRecord struct {
@@ -375,6 +387,32 @@ func TestListBackupsAndFindBackupSelections(t *testing.T) {
 	}
 }
 
+func TestDeleteBackupUsesAdvertisedAction(t *testing.T) {
+	ctx := context.Background()
+	serverURL := ""
+	var deleteBody string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		raw, _ := io.ReadAll(r.Body)
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/v1/backupvolumes/vol-a":
+			_, _ = w.Write([]byte(`{"name":"vol-a","actions":{"backupDelete":"` + serverURL + `/delete"}}`))
+		case r.Method == http.MethodPost && r.URL.Path == "/delete":
+			deleteBody = string(raw)
+			_, _ = w.Write([]byte(`{"name":"vol-a"}`))
+		default:
+			t.Fatalf("unexpected request: %s %s body=%s", r.Method, r.URL.String(), string(raw))
+		}
+	}))
+	defer server.Close()
+	serverURL = server.URL
+
+	client := New(server.URL)
+	if err := client.DeleteBackup(ctx, "vol-a", "backup-old"); err != nil {
+		t.Fatalf("DeleteBackup: %v", err)
+	}
+	assertBodyContains(t, deleteBody, `"name":"backup-old"`)
+}
+
 func TestListBackupsAndFindBackupErrors(t *testing.T) {
 	ctx := context.Background()
 
@@ -391,6 +429,19 @@ func TestListBackupsAndFindBackupErrors(t *testing.T) {
 		}
 	})
 
+	t.Run("missing backupDelete action", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write([]byte(`{"name":"vol-a","actions":{}}`))
+		}))
+		defer server.Close()
+
+		client := New(server.URL)
+		err := client.DeleteBackup(ctx, "vol-a", "backup-a")
+		if err == nil || !strings.Contains(err.Error(), "backup delete action missing") {
+			t.Fatalf("expected missing backupDelete action error, got %v", err)
+		}
+	})
+
 	t.Run("backup volume lookup error", func(t *testing.T) {
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, "volume lookup failed", http.StatusBadGateway)

internal/server/backup_handlers.go

@@ -232,6 +232,9 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
 		if !ready {
 			return api.BackupResponse{}, "not_ready", errors.New(reason)
 		}
+		if err := s.pruneLonghornBackups(ctx, volumeName, req.Namespace, req.PVC, resolvedKeepLast); err != nil {
+			return api.BackupResponse{}, "backend_error", err
+		}
 
 		labels := map[string]string{
 			"soteria.bstein.dev/namespace":    req.Namespace,
@@ -244,6 +247,9 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
 		if _, err := s.longhorn.SnapshotBackup(ctx, volumeName, backupID, labels, s.cfg.LonghornBackupMode); err != nil {
 			return api.BackupResponse{}, "backend_error", err
 		}
+		if err := s.pruneLonghornBackups(ctx, volumeName, req.Namespace, req.PVC, resolvedKeepLast); err != nil {
+			return api.BackupResponse{}, "backend_error", err
+		}
 		return response, "success", nil
 	case "restic":
 		_, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)

internal/server/backup_handlers_test.go

@@ -32,6 +32,15 @@ func assertErrorResponseContains(t *testing.T, res *httptest.ResponseRecorder, w
 	}
 }
 
+func stringSliceContains(items []string, want string) bool {
+	for _, item := range items {
+		if item == want {
+			return true
+		}
+	}
+	return false
+}
+
 type backupTestKubeClient struct {
 	*restoreTestKubeClient
 	createBackupErr      error
@@ -475,6 +484,60 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
 		}
 	})
 
+	t.Run("longhorn prunes soteria backups by keep last", func(t *testing.T) {
+		keepLast := 2
+		fakeLonghorn := &fakeLonghornClient{backups: []longhorn.Backup{
+			{
+				Name:    "backup-keep-new",
+				Created: "2026-04-20T10:00:00Z",
+				State:   "Completed",
+				Labels:  map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"},
+			},
+			{
+				Name:    "backup-keep-mid",
+				Created: "2026-04-19T10:00:00Z",
+				State:   "Completed",
+				Labels:  map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"},
+			},
+			{
+				Name:    "backup-delete-old",
+				Created: "2026-04-18T10:00:00Z",
+				State:   "Completed",
+				Labels:  map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"},
+			},
+			{
+				Name:    "backup-delete-failed",
+				Created: "2026-04-21T10:00:00Z",
+				State:   "Error",
+				Labels:  map[string]string{longhornLabelNamespace: "apps", longhornLabelPVC: "data"},
+			},
+			{
+				Name:    "backup-foreign",
+				Created: "2026-04-17T10:00:00Z",
+				State:   "Completed",
+				Labels:  map[string]string{longhornLabelNamespace: "other", longhornLabelPVC: "data"},
+			},
+		}}
+		srv := newBackupTestServer(
+			&config.Config{AuthRequired: false, BackupDriver: "longhorn"},
+			&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{}}},
+			&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: fakeLonghorn}},
+		)
+		_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data", KeepLast: &keepLast}, "brad")
+		if result != "success" || err != nil {
+			t.Fatalf("expected longhorn backup success with pruning, got result=%q err=%v", result, err)
+		}
+		if !stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-delete-old") {
+			t.Fatalf("expected old Soteria backup to be deleted, got %#v", fakeLonghorn.deletedBackups)
+		}
+		if !stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-delete-failed") {
+			t.Fatalf("expected failed Soteria backup to be deleted, got %#v", fakeLonghorn.deletedBackups)
+		}
+		if stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-keep-new") || stringSliceContains(fakeLonghorn.deletedBackups, "apps-data-pv/backup-foreign") {
+			t.Fatalf("expected newest and foreign backups to be retained, got %#v", fakeLonghorn.deletedBackups)
+		}
+	})
+
 	t.Run("longhorn detached volume", func(t *testing.T) {
 		srv := newBackupTestServer(
 			&config.Config{AuthRequired: false, BackupDriver: "longhorn"},

internal/server/longhorn_retention.go

@@ -0,0 +1,84 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/soteria/internal/longhorn"
+)
+
+const (
+	longhornLabelNamespace = "soteria.bstein.dev/namespace"
+	longhornLabelPVC       = "soteria.bstein.dev/pvc"
+)
+
+func (s *Server) pruneLonghornBackups(ctx context.Context, volumeName, namespace, pvc string, keepLast int) error {
+	if keepLast <= 0 {
+		return nil
+	}
+
+	backups, err := s.longhorn.ListBackups(ctx, volumeName)
+	if err != nil {
+		if longhorn.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	completed := make([]longhorn.Backup, 0, len(backups))
+	toDelete := []longhorn.Backup{}
+	for _, backup := range backups {
+		if !longhornBackupMatchesPVC(backup, namespace, pvc) {
+			continue
+		}
+		switch strings.ToLower(strings.TrimSpace(backup.State)) {
+		case "completed":
+			completed = append(completed, backup)
+		case "error", "failed":
+			toDelete = append(toDelete, backup)
+		}
+	}
+
+	sort.Slice(completed, func(i, j int) bool {
+		left := longhornBackupCreatedAt(completed[i])
+		right := longhornBackupCreatedAt(completed[j])
+		if left.Equal(right) {
+			return completed[i].Name > completed[j].Name
+		}
+		return left.After(right)
+	})
+
+	if len(completed) > keepLast {
+		toDelete = append(toDelete, completed[keepLast:]...)
+	}
+
+	for _, backup := range toDelete {
+		if strings.TrimSpace(backup.Name) == "" {
+			continue
+		}
+		if err := s.longhorn.DeleteBackup(ctx, volumeName, backup.Name); err != nil {
+			return fmt.Errorf("delete old Longhorn backup %s for %s/%s: %w", backup.Name, namespace, pvc, err)
+		}
+	}
+
+	return nil
+}
+
+func longhornBackupMatchesPVC(backup longhorn.Backup, namespace, pvc string) bool {
+	return backup.Labels[longhornLabelNamespace] == namespace && backup.Labels[longhornLabelPVC] == pvc
+}
+
+func longhornBackupCreatedAt(backup longhorn.Backup) time.Time {
+	created := strings.TrimSpace(backup.Created)
+	if created == "" {
+		return time.Time{}
+	}
+	ts, err := time.Parse(time.RFC3339, created)
+	if err != nil {
+		return time.Time{}
+	}
+	return ts
+}

internal/server/server.go

@@ -39,6 +39,7 @@ type longhornClient interface {
 	DeleteVolume(ctx context.Context, volumeName string) error
 	FindBackup(ctx context.Context, volumeName, snapshot string) (*longhorn.Backup, error)
 	ListBackups(ctx context.Context, volumeName string) ([]longhorn.Backup, error)
+	DeleteBackup(ctx context.Context, volumeName, backupName string) error
 }
 
 // Server owns HTTP routing, policy state, telemetry, and the UI renderer.

internal/server/server_test.go

@@ -176,6 +176,7 @@ func (f *secretErrorKubeClient) LoadSecretData(_ context.Context, _, _, _ string
 type fakeLonghornClient struct {
 	backups            []longhorn.Backup
 	volumes            map[string]longhorn.Volume
+	deletedBackups     []string
 	createSnapshotName string
 	snapshotBackupName string
 }
@@ -221,6 +222,18 @@ func (f *fakeLonghornClient) ListBackups(_ context.Context, volumeName string) (
 	return f.backups, nil
 }
 
+func (f *fakeLonghornClient) DeleteBackup(_ context.Context, volumeName, backupName string) error {
+	f.deletedBackups = append(f.deletedBackups, volumeName+"/"+backupName)
+	filtered := f.backups[:0]
+	for _, backup := range f.backups {
+		if backup.Name != backupName {
+			filtered = append(filtered, backup)
+		}
+	}
+	f.backups = filtered
+	return nil
+}
+
 func TestNewInitializesCoreServerFields(t *testing.T) {
 	srv := New(&config.Config{}, nil, nil)