bstein/soteria
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backup: harden restic policy jobs

Browse Source
This commit is contained in:
codex 2026-07-16 01:05:33 -03:00
parent 73525ad712
commit fdab56ed82
23 changed files with 802 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
internal/config/config.go
View File

@ -13,7 +13,8 @@ import (
const (
defaultBackupDriver = "longhorn"
defaultResticImage = "restic/restic:0.16.4"
defaultJobTTLSeconds = 86400
defaultJobTTLSeconds = 259200
defaultJobBackoff = 1
defaultListenAddr = ":8080"
defaultResticSecret = "soteria-restic"
defaultLonghornURL = "http://longhorn-backend.longhorn-system.svc:9500"
@ -26,6 +27,7 @@ const (
defaultUsageSecret = "soteria-backup-usage"
defaultB2ScanInterval = 15 * time.Minute
defaultB2ScanTimeout = 2 * time.Minute
defaultExcludedSCs = "local-path"
serviceNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
)
@ -39,14 +41,19 @@ type Config struct {
ResticSecretName string
ResticBackupArgs []string
ResticForgetArgs []string
ResticPruneAfterBackup bool
ResticUnlockBeforeBackup bool
S3Endpoint string
S3Region string
JobTTLSeconds int32
JobBackoffLimit int32
JobNodeSelector map[string]string
JobCPURequest string
JobCPULimit string
JobMemoryRequest string
JobMemoryLimit string
ExcludedStorageClasses []string
ExcludedPVCs []string
WorkerServiceAccount string
ListenAddr string
LonghornURL string
@ -99,6 +106,8 @@ func Load() (*Config, error) {
cfg.ResticSecretName = getenvDefault("SOTERIA_RESTIC_SECRET_NAME", defaultResticSecret)
cfg.ResticBackupArgs = strings.Fields(getenv("SOTERIA_RESTIC_BACKUP_ARGS"))
cfg.ResticForgetArgs = strings.Fields(getenv("SOTERIA_RESTIC_FORGET_ARGS"))
cfg.ResticPruneAfterBackup = getenvBool("SOTERIA_RESTIC_PRUNE_AFTER_BACKUP")
cfg.ResticUnlockBeforeBackup = getenvBool("SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP")
cfg.S3Endpoint = getenv("SOTERIA_S3_ENDPOINT")
cfg.S3Region = getenv("SOTERIA_S3_REGION")
cfg.WorkerServiceAccount = getenv("SOTERIA_JOB_SERVICE_ACCOUNT")
@ -108,6 +117,8 @@ func Load() (*Config, error) {
cfg.JobCPULimit = getenv("SOTERIA_JOB_CPU_LIMIT")
cfg.JobMemoryRequest = getenv("SOTERIA_JOB_MEMORY_REQUEST")
cfg.JobMemoryLimit = getenv("SOTERIA_JOB_MEMORY_LIMIT")
cfg.ExcludedStorageClasses = parseCSV(getenvDefault("SOTERIA_EXCLUDED_STORAGE_CLASSES", defaultExcludedSCs))
cfg.ExcludedPVCs = parseCSV(getenv("SOTERIA_EXCLUDED_PVCS"))
cfg.LonghornURL = getenvDefault("SOTERIA_LONGHORN_URL", defaultLonghornURL)
cfg.LonghornBackupMode = getenvDefault("SOTERIA_LONGHORN_BACKUP_MODE", defaultLonghornMode)
cfg.AuthRequired = getenvBool("SOTERIA_AUTH_REQUIRED")
@ -137,6 +148,11 @@ func Load() (*Config, error) {
} else {
cfg.JobTTLSeconds = defaultJobTTLSeconds
}
if retries, ok := getenvInt("SOTERIA_JOB_BACKOFF_LIMIT"); ok {
cfg.JobBackoffLimit = int32(retries)
} else {
cfg.JobBackoffLimit = defaultJobBackoff
}
if seconds, ok := getenvInt("SOTERIA_METRICS_REFRESH_SECONDS"); ok {
cfg.MetricsRefreshInterval = time.Duration(seconds) * time.Second
@ -182,6 +198,12 @@ func Load() (*Config, error) {
if cfg.JobNodeSelector == nil {
return nil, errors.New("SOTERIA_JOB_NODE_SELECTOR is invalid; expected key=value pairs")
}
if cfg.JobTTLSeconds <= 0 {
return nil, errors.New("SOTERIA_JOB_TTL_SECONDS must be greater than zero")
}
if cfg.JobBackoffLimit < 0 {
return nil, errors.New("SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero")
}
if err := validateQuantity("SOTERIA_JOB_CPU_REQUEST", cfg.JobCPURequest); err != nil {
return nil, err
}

39
internal/config/config_test.go
View File

@ -22,13 +22,18 @@ func clearConfigEnv(t *testing.T) {
"SOTERIA_RESTIC_SECRET_NAME",
"SOTERIA_RESTIC_BACKUP_ARGS",
"SOTERIA_RESTIC_FORGET_ARGS",
"SOTERIA_RESTIC_PRUNE_AFTER_BACKUP",
"SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP",
"SOTERIA_S3_ENDPOINT",
"SOTERIA_S3_REGION",
"SOTERIA_JOB_SERVICE_ACCOUNT",
"SOTERIA_LISTEN_ADDR",
"SOTERIA_JOB_NODE_SELECTOR",
"SOTERIA_JOB_BACKOFF_LIMIT",
"SOTERIA_LONGHORN_URL",
"SOTERIA_LONGHORN_BACKUP_MODE",
"SOTERIA_EXCLUDED_STORAGE_CLASSES",
"SOTERIA_EXCLUDED_PVCS",
"SOTERIA_AUTH_REQUIRED",
"SOTERIA_ALLOWED_GROUPS",
"SOTERIA_AUTH_BEARER_TOKENS",
@ -82,6 +87,12 @@ func TestLoadDefaultsForLonghorn(t *testing.T) {
if cfg.JobTTLSeconds != defaultJobTTLSeconds {
t.Fatalf("expected default ttl %d, got %d", defaultJobTTLSeconds, cfg.JobTTLSeconds)
}
if cfg.JobBackoffLimit != defaultJobBackoff {
t.Fatalf("expected default backoff %d, got %d", defaultJobBackoff, cfg.JobBackoffLimit)
}
if len(cfg.ExcludedStorageClasses) != 1 || cfg.ExcludedStorageClasses[0] != "local-path" {
t.Fatalf("expected local-path to be excluded by default, got %#v", cfg.ExcludedStorageClasses)
}
if cfg.MetricsRefreshInterval != defaultMetricsRefresh || cfg.PolicyEvalInterval != defaultPolicyEval {
t.Fatalf("unexpected interval defaults: %#v", cfg)
}
@ -108,6 +119,8 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
withEnv(t, "SOTERIA_RESTIC_SECRET_NAME", "restic-creds")
withEnv(t, "SOTERIA_RESTIC_BACKUP_ARGS", "--one-file-system --exclude /tmp")
withEnv(t, "SOTERIA_RESTIC_FORGET_ARGS", "--keep-last 5 --prune")
withEnv(t, "SOTERIA_RESTIC_PRUNE_AFTER_BACKUP", "true")
withEnv(t, "SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP", "true")
withEnv(t, "SOTERIA_S3_ENDPOINT", "https://b2.example.invalid")
withEnv(t, "SOTERIA_S3_REGION", "us-west-000")
withEnv(t, "SOTERIA_JOB_SERVICE_ACCOUNT", "soteria-worker")
@ -115,6 +128,9 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
withEnv(t, "SOTERIA_JOB_CPU_LIMIT", "1")
withEnv(t, "SOTERIA_JOB_MEMORY_REQUEST", "256Mi")
withEnv(t, "SOTERIA_JOB_MEMORY_LIMIT", "2Gi")
withEnv(t, "SOTERIA_JOB_BACKOFF_LIMIT", "2")
withEnv(t, "SOTERIA_EXCLUDED_STORAGE_CLASSES", "local-path, ephemeral")
withEnv(t, "SOTERIA_EXCLUDED_PVCS", "jenkins/jenkins-dind-cache, apps/tmp-*")
withEnv(t, "SOTERIA_LISTEN_ADDR", ":9090")
withEnv(t, "SOTERIA_JOB_NODE_SELECTOR", "hardware=rpi5,role=worker")
withEnv(t, "SOTERIA_AUTH_REQUIRED", "true")
@ -156,12 +172,21 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
if strings.Join(cfg.ResticForgetArgs, " ") != "--keep-last 5 --prune" {
t.Fatalf("unexpected forget args: %#v", cfg.ResticForgetArgs)
}
if !cfg.ResticPruneAfterBackup || !cfg.ResticUnlockBeforeBackup {
t.Fatalf("expected explicit restic safety overrides, got prune=%v unlock=%v", cfg.ResticPruneAfterBackup, cfg.ResticUnlockBeforeBackup)
}
if cfg.WorkerServiceAccount != "soteria-worker" || cfg.ListenAddr != ":9090" {
t.Fatalf("unexpected worker/listen config: %#v", cfg)
}
if cfg.JobCPURequest != "100m" || cfg.JobCPULimit != "1" || cfg.JobMemoryRequest != "256Mi" || cfg.JobMemoryLimit != "2Gi" {
t.Fatalf("unexpected job resources: %#v", cfg)
}
if cfg.JobBackoffLimit != 2 {
t.Fatalf("unexpected job backoff: %#v", cfg.JobBackoffLimit)
}
if strings.Join(cfg.ExcludedStorageClasses, ",") != "local-path,ephemeral" || strings.Join(cfg.ExcludedPVCs, ",") != "jenkins/jenkins-dind-cache,apps/tmp-*" {
t.Fatalf("unexpected exclusions: storage=%#v pvcs=%#v", cfg.ExcludedStorageClasses, cfg.ExcludedPVCs)
}
if cfg.JobNodeSelector["hardware"] != "rpi5" || cfg.JobNodeSelector["role"] != "worker" {
t.Fatalf("unexpected node selector: %#v", cfg.JobNodeSelector)
}
@ -265,6 +290,20 @@ func TestLoadRejectsInvalidConfigurations(t *testing.T) {
},
substr: "SOTERIA_BACKUP_MAX_AGE_HOURS must be greater than zero",
},
{
name: "invalid job ttl",
env: map[string]string{
"SOTERIA_JOB_TTL_SECONDS": "0",
},
substr: "SOTERIA_JOB_TTL_SECONDS must be greater than zero",
},
{
name: "invalid job backoff",
env: map[string]string{
"SOTERIA_JOB_BACKOFF_LIMIT": "-1",
},
substr: "SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero",
},
{
name: "b2 enabled without endpoint or secret",
env: map[string]string{

13
internal/k8s/job_helpers_test.go
View File

@ -82,7 +82,7 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
Dedupe: &dedupe,
KeepLast: &keepLast,
})
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
if strings.Contains(cmd, "restic unlock") || strings.Contains(cmd, "--prune") || !strings.Contains(cmd, "set -eu;") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
t.Fatalf("expected backup command to include bootstrap/dedupe/keep-last logic, got %q", cmd)
}
if !strings.Contains(cmd, "--exclude *.tmp") || !strings.Contains(cmd, "--tag nightly") || !strings.Contains(cmd, "--tag prod") {
@ -94,6 +94,17 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
t.Fatalf("expected backup command to use configured forget args, got %q", cmd)
}
cfg.ResticPruneAfterBackup = true
cfg.ResticUnlockBeforeBackup = true
cfg.ResticForgetArgs = nil
cmd = backupCommand(cfg, api.BackupRequest{PVC: "data", KeepLast: &keepLast})
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "--keep-last 3 --prune") {
t.Fatalf("expected explicit unlock/prune opts to be honored, got %q", cmd)
}
if !strings.Contains(cmd, "ensure_restic_repo") || !strings.Contains(cmd, "already initialized") {
t.Fatalf("expected robust restic bootstrap handling, got %q", cmd)
}
env := resticEnv(cfg, "restic-secret", "")
values := map[string]string{}
for _, item := range env {

27
internal/k8s/job_manifests.go
View File

@ -80,7 +80,7 @@ func buildBackupJob(cfg *config.Config, req api.BackupRequest, jobName, secretNa
Annotations: annotations,
},
Spec: batchv1.JobSpec{
BackoffLimit: int32Ptr(0),
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
@ -161,7 +161,7 @@ func buildRestoreJob(cfg *config.Config, req api.RestoreTestRequest, jobName, se
Annotations: annotations,
},
Spec: batchv1.JobSpec{
BackoffLimit: int32Ptr(0),
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
@ -216,27 +216,38 @@ func backupCommand(cfg *config.Config, req api.BackupRequest) string {
cmd := strings.Join(args, " ")
if keepLast > 0 {
forget := strings.Join([]string{
forgetArgs := []string{
"restic", "forget",
"--host", "soteria",
"--group-by", "host,tags",
"--tag", fmt.Sprintf("pvc=%s", req.PVC),
"--keep-last", strconv.Itoa(keepLast),
"--prune",
}, " ")
}
if cfg.ResticPruneAfterBackup {
forgetArgs = append(forgetArgs, "--prune")
}
forget := strings.Join(forgetArgs, " ")
cmd = fmt.Sprintf("%s && %s", cmd, forget)
} else if len(cfg.ResticForgetArgs) > 0 {
forget := strings.Join(append([]string{"restic", "forget"}, cfg.ResticForgetArgs...), " ")
cmd = fmt.Sprintf("%s && %s", cmd, forget)
}
bootstrap := "restic unlock >/dev/null 2>&1 || true; restic cat config >/dev/null 2>&1 || restic init; restic snapshots >/dev/null 2>&1 || restic rebuild-index"
return "set -euo pipefail; " + bootstrap + "; " + cmd
bootstrapSteps := []string{resticRepoBootstrapCommand(), "ensure_restic_repo", "restic snapshots >/dev/null 2>&1 || restic rebuild-index"}
if cfg.ResticUnlockBeforeBackup {
bootstrapSteps = append([]string{"restic unlock >/dev/null 2>&1 || true"}, bootstrapSteps...)
}
bootstrap := strings.Join(bootstrapSteps, "; ")
return "set -eu; " + bootstrap + "; " + cmd
}
func resticRepoBootstrapCommand() string {
return `ensure_restic_repo() { probe_log="$(mktemp)"; if restic cat config >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if restic init >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if grep -qi "already initialized" "${probe_log}"; then rm -f "${probe_log}"; return 0; fi; cat "${probe_log}" >&2; rm -f "${probe_log}"; return 1; }`
}
func restoreCommand(snapshot string) string {
return fmt.Sprintf(
"set -euo pipefail; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
"set -eu; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
snapshot,
)
}

8
internal/k8s/job_manifests_test.go
View File

@ -19,6 +19,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
S3Region: "us-west-001",
JobTTLSeconds: 3600,
JobBackoffLimit: 1,
JobNodeSelector: map[string]string{"hardware": "rpi5"},
JobCPURequest: "100m",
JobCPULimit: "1",
@ -55,7 +56,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
if container.Name != "restic" || container.Image != "restic/restic:latest" {
t.Fatalf("expected restic container, got %#v", container)
}
if len(container.Args) != 1 || !strings.Contains(container.Args[0], "restic unlock") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
if len(container.Args) != 1 || strings.Contains(container.Args[0], "restic unlock") || strings.Contains(container.Args[0], "--prune") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
t.Fatalf("expected backup command payload, got %#v", container.Args)
}
if got := container.Resources.Requests[corev1.ResourceCPU]; got.String() != "100m" {
@ -67,8 +68,8 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
if len(job.Spec.Template.Spec.Volumes) != 2 || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim == nil {
t.Fatalf("expected pvc + cache volumes, got %#v", job.Spec.Template.Spec.Volumes)
}
if got := *job.Spec.BackoffLimit; got != 0 {
t.Fatalf("expected zero backoff limit, got %d", got)
if got := *job.Spec.BackoffLimit; got != 1 {
t.Fatalf("expected configured backoff limit, got %d", got)
}
if got := *job.Spec.TTLSecondsAfterFinished; got != 3600 {
t.Fatalf("expected ttl from config, got %d", got)
@ -82,6 +83,7 @@ func TestBuildRestoreJobCoversDefaultAndTargetPVCRestoreShapes(t *testing.T) {
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
S3Region: "us-west-001",
JobTTLSeconds: 1800,
JobBackoffLimit: 1,
JobNodeSelector: map[string]string{"hardware": "rpi5"},
WorkerServiceAccount: "soteria-worker",
}

5
internal/k8s/job_support.go
View File

@ -109,6 +109,11 @@ func resticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string
return appendRepositoryPath(base, suffix)
}
// ResticRepositoryForBackup returns the repository path Soteria will use for a PVC backup.
func ResticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string {
return resticRepositoryForBackup(base, namespace, pvc, dedupe)
}
func sanitizeRepositorySegment(value string) string {
sanitized := sanitizeName(value)
if sanitized == "" {

1
internal/k8s/jobs.go
View File

@ -199,6 +199,7 @@ func (c *Client) CreateBackupJob(ctx context.Context, cfg *config.Config, req ap
job := buildBackupJob(cfg, req, jobName, secretName, repository, dedupeEnabled, keepLast)
if nodeName, err := c.resolvePVCMountedNode(ctx, req.Namespace, req.PVC); err == nil && nodeName != "" {
job.Spec.Template.Spec.NodeName = nodeName
job.Spec.Template.Spec.NodeSelector = nil
}
created, err := c.Clientset.BatchV1().Jobs(req.Namespace).Create(ctx, job, metav1.CreateOptions{})
if err != nil {

86
internal/k8s/jobs_test.go
View File

@ -190,6 +190,88 @@ func TestResolvePVCMountedNodeIgnoresDeadPodsAndFindsMountedClaim(t *testing.T)
}
}
func TestListPVCMountsMarksSoteriaAndIgnoresDeadPods(t *testing.T) {
now := metav1.NewTime(time.Now().UTC())
client := &Client{Clientset: k8sfake.NewSimpleClientset(
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "deleted", Namespace: "apps", DeletionTimestamp: &now},
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}}},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "failed", Namespace: "apps"},
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}}},
Status: corev1.PodStatus{Phase: corev1.PodFailed},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "apps"},
Spec: corev1.PodSpec{
NodeName: "titan-02",
Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "soteria-backup-data",
Namespace: "apps",
Labels: map[string]string{
labelAppName: "soteria",
labelComponent: "backup",
},
},
Spec: corev1.PodSpec{
NodeName: "titan-02",
Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
)}
mounts, err := client.ListPVCMounts(context.Background(), "apps", "data")
if err != nil {
t.Fatalf("list pvc mounts: %v", err)
}
if len(mounts) != 2 {
t.Fatalf("expected app and soteria mounts only, got %#v", mounts)
}
if mounts[0].PodName != "app" || mounts[0].NodeName != "titan-02" || mounts[0].SoteriaBackup {
t.Fatalf("expected first mount to be the app pod, got %#v", mounts[0])
}
if mounts[1].PodName != "soteria-backup-data" || !mounts[1].SoteriaBackup {
t.Fatalf("expected second mount to be marked as soteria, got %#v", mounts[1])
}
clientset := k8sfake.NewSimpleClientset()
clientset.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
return true, nil, errors.New("list mounts exploded")
})
client = &Client{Clientset: clientset}
if _, err := client.ListPVCMounts(context.Background(), "apps", "data"); err == nil || !strings.Contains(err.Error(), "list mounts exploded") {
t.Fatalf("expected list pvc mounts error, got %v", err)
}
}
func TestReadBackupJobLogCoversSuccessAndListFailures(t *testing.T) {
client := &Client{Clientset: k8sfake.NewSimpleClientset(
&corev1.Pod{
@ -301,6 +383,7 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
ResticRepository: "s3:https://repo/root",
ResticImage: "restic/restic:latest",
JobTTLSeconds: 3600,
JobNodeSelector: map[string]string{"kubernetes.io/arch": "arm64"},
WorkerServiceAccount: "soteria-sa",
}
@ -345,6 +428,9 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
if job.Spec.Template.Spec.NodeName != "titan-02" || job.Spec.Template.Spec.ServiceAccountName != "soteria-sa" {
t.Fatalf("expected node pin + service account, got %#v", job.Spec.Template.Spec)
}
if len(job.Spec.Template.Spec.NodeSelector) != 0 {
t.Fatalf("expected node selector to be cleared when backup is pinned to mounted PVC node, got %#v", job.Spec.Template.Spec.NodeSelector)
}
if job.Annotations[annotationDedupeEnabled] != "false" || job.Annotations[annotationKeepLast] != "3" {
t.Fatalf("expected backup annotations, got %#v", job.Annotations)
}

40
internal/k8s/volumes.go
View File

@ -21,6 +21,14 @@ type PVCSummary struct {
AccessModes []string
}
// PVCMount describes an active pod mounting a PVC.
type PVCMount struct {
PodName string
NodeName string
Phase string
SoteriaBackup bool
}
// ResolvePVCVolume loads a PVC and its bound PV so backup handlers can act on the real volume.
func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
@ -39,6 +47,38 @@ func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string
return pvc.Spec.VolumeName, pvc, pv, nil
}
// ListPVCMounts returns active pods in a namespace that reference the PVC.
func (c *Client) ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]PVCMount, error) {
pods, err := c.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("list pods for pvc %s/%s: %w", namespace, pvcName, err)
}
mounts := []PVCMount{}
for _, pod := range pods.Items {
if pod.DeletionTimestamp != nil {
continue
}
if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
continue
}
for _, volume := range pod.Spec.Volumes {
claim := volume.PersistentVolumeClaim
if claim == nil || claim.ClaimName != pvcName {
continue
}
mounts = append(mounts, PVCMount{
PodName: pod.Name,
NodeName: pod.Spec.NodeName,
Phase: string(pod.Status.Phase),
SoteriaBackup: pod.Labels[labelAppName] == "soteria" && pod.Labels[labelComponent] == "backup",
})
break
}
}
return mounts, nil
}
// ListBoundPVCs returns all bound PVCs in a stable namespace/name order.
func (c *Client) ListBoundPVCs(ctx context.Context) ([]PVCSummary, error) {
list, err := c.Clientset.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{})

39
internal/server/backup_handlers.go
View File

@ -3,6 +3,7 @@ package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
@ -238,6 +239,34 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
}
return response, "success", nil
case "restic":
_, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
if err != nil {
return api.BackupResponse{}, "validation_error", err
}
storageClass := ""
if pvc != nil && pvc.Spec.StorageClassName != nil {
storageClass = *pvc.Spec.StorageClassName
}
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
blocked, reason, err := s.liveExclusivePVCMounted(ctx, req.Namespace, req.PVC, pvcAccessModes(pvc))
if err != nil {
return api.BackupResponse{}, "backend_error", err
}
if blocked {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, req.Namespace, req.PVC, resolvedDedupe)
if !req.DryRun {
busy, err := s.resticRepositoryBusy(ctx, repository)
if err != nil {
return api.BackupResponse{}, "backend_error", err
}
if busy {
return api.BackupResponse{}, "in_progress", fmt.Errorf("restic repository already has an active Soteria backup")
}
}
jobName, secretName, err := s.client.CreateBackupJob(ctx, s.cfg, req)
if err != nil {
return api.BackupResponse{}, "backend_error", err
@ -268,9 +297,13 @@ func (s *Server) listNamespaceBoundPVCs(ctx context.Context, namespace string) (
}
filtered := make([]k8s.PVCSummary, 0, len(items))
for _, item := range items {
if item.Namespace == namespace {
filtered = append(filtered, item)
if item.Namespace != namespace {
continue
}
if excluded, _ := s.pvcSummaryExcluded(item); excluded {
continue
}
filtered = append(filtered, item)
}
return filtered, nil
}
@ -279,6 +312,8 @@ func backupStatusCode(result string) int {
switch result {
case "validation_error", "unsupported_driver":
return http.StatusBadRequest
case "in_progress":
return http.StatusConflict
case "backend_error":
return http.StatusBadGateway
default:

115
internal/server/backup_handlers_test.go
View File

@ -354,6 +354,69 @@ func TestHandleBackupAndNamespaceBackupValidationPaths(t *testing.T) {
})
}
func TestHandleBackupReturnsExecutionErrors(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"ops/other": {
{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(`{"namespace":"apps","pvc":"data"}`))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
assertErrorResponseContains(t, res, http.StatusConflict, "restic repository already has an active")
}
func TestHandleNamespaceBackupReportsPartialFailureAndFiltersExcludedPVCs(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "alpha", VolumeName: "pv-alpha", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "beta", VolumeName: "pv-beta", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
}},
createBackupErrForPVC: map[string]error{"beta": errors.New("create beta backup exploded")},
}
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root", ExcludedStorageClasses: []string{"local-path"}},
client,
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(`{"namespace":"apps","dedupe":false}`))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.NamespaceBackupResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode payload: %v", err)
}
if payload.Total != 2 || payload.Succeeded != 1 || payload.Failed != 1 {
t.Fatalf("expected excluded cache to be filtered and beta failure reported, got %#v", payload)
}
if len(payload.Results) != 2 || payload.Results[0].PVC != "alpha" || payload.Results[1].PVC != "beta" || payload.Results[1].Status != "backend_error" {
t.Fatalf("unexpected namespace backup results: %#v", payload.Results)
}
}
func TestExecuteBackupAndStatusHelpers(t *testing.T) {
t.Run("validation error", func(t *testing.T) {
srv := newBackupTestServer(
@ -427,6 +490,57 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
}
})
t.Run("restic excluded pvc", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ExcludedStorageClasses: []string{"local-path"}},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"}},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "cache"}, "brad")
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "storage class local-path is excluded") {
t.Fatalf("expected excluded pvc validation error, got result=%q err=%v", result, err)
}
})
t.Run("restic live rwo pvc", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
pvcMounts: map[string][]k8s.PVCMount{
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "mounted by active pod gitea-0") {
t.Fatalf("expected live RWO validation error, got result=%q err=%v", result, err)
}
})
t.Run("restic active repository conflict", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"ops/other": {{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()}},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
if result != "in_progress" || err == nil || !strings.Contains(err.Error(), "restic repository already has an active") {
t.Fatalf("expected active repository conflict, got result=%q err=%v", result, err)
}
})
t.Run("unsupported driver", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "mystery"},
@ -443,6 +557,7 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
testCases := map[string]int{
"validation_error": http.StatusBadRequest,
"unsupported_driver": http.StatusBadRequest,
"in_progress": http.StatusConflict,
"backend_error": http.StatusBadGateway,
"other": http.StatusInternalServerError,
}

147
internal/server/backup_safety.go Normal file
View File

@ -0,0 +1,147 @@
package server
import (
"context"
"fmt"
"path"
"sort"
"strings"
"scm.bstein.dev/bstein/soteria/internal/k8s"
corev1 "k8s.io/api/core/v1"
)
func (s *Server) pvcSummaryExcluded(pvc k8s.PVCSummary) (bool, string) {
return s.pvcExcluded(pvc.Namespace, pvc.Name, pvc.StorageClass)
}
func (s *Server) pvcExcluded(namespace, pvc, storageClass string) (bool, string) {
storageClass = strings.TrimSpace(storageClass)
for _, excluded := range s.cfg.ExcludedStorageClasses {
if storageClass != "" && strings.EqualFold(storageClass, strings.TrimSpace(excluded)) {
return true, fmt.Sprintf("storage class %s is excluded from restic backups", storageClass)
}
}
key := strings.Trim(strings.TrimSpace(namespace)+"/"+strings.TrimSpace(pvc), "/")
for _, pattern := range s.cfg.ExcludedPVCs {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
continue
}
if pattern == key {
return true, fmt.Sprintf("PVC %s is excluded from restic backups", key)
}
matched, err := path.Match(pattern, key)
if err == nil && matched {
return true, fmt.Sprintf("PVC %s matches excluded pattern %s", key, pattern)
}
}
return false, ""
}
func (s *Server) liveExclusivePVCMounted(ctx context.Context, namespace, pvc string, accessModes []string) (bool, string, error) {
if !hasExclusiveAccessMode(accessModes) {
return false, "", nil
}
mounts, err := s.client.ListPVCMounts(ctx, namespace, pvc)
if err != nil {
return false, "", err
}
for _, mount := range mounts {
if mount.SoteriaBackup {
continue
}
location := strings.TrimSpace(mount.PodName)
if mount.NodeName != "" {
location += " on " + mount.NodeName
}
return true, fmt.Sprintf("RWO PVC %s/%s is mounted by active pod %s; restic backups wait for the workload to release the claim or use Longhorn snapshots", namespace, pvc, location), nil
}
return false, "", nil
}
func hasExclusiveAccessMode(accessModes []string) bool {
for _, mode := range accessModes {
switch corev1.PersistentVolumeAccessMode(strings.TrimSpace(mode)) {
case corev1.ReadWriteOnce, corev1.ReadWriteOncePod:
return true
}
}
return false
}
func pvcAccessModes(pvc *corev1.PersistentVolumeClaim) []string {
if pvc == nil {
return nil
}
modes := make([]string, 0, len(pvc.Spec.AccessModes))
for _, mode := range pvc.Spec.AccessModes {
modes = append(modes, string(mode))
}
return modes
}
func (s *Server) activeResticRepositories(ctx context.Context, namespaces []string) (map[string]struct{}, error) {
active := map[string]struct{}{}
for _, namespace := range uniqueSortedStrings(namespaces) {
jobs, err := s.client.ListBackupJobs(ctx, namespace)
if err != nil {
return nil, err
}
for _, job := range jobs {
if !backupJobInProgress(job.State) {
continue
}
repository := strings.TrimSpace(job.Repository)
if repository == "" {
repository = strings.TrimSpace(s.cfg.ResticRepository)
}
if repository != "" {
active[repository] = struct{}{}
}
}
}
return active, nil
}
func (s *Server) resticRepositoryBusy(ctx context.Context, repository string) (bool, error) {
repository = strings.TrimSpace(repository)
if repository == "" {
return false, nil
}
pvcs, err := s.client.ListBoundPVCs(ctx)
if err != nil {
return false, err
}
namespaces := make([]string, 0, len(pvcs))
for _, pvc := range pvcs {
namespaces = append(namespaces, pvc.Namespace)
}
active, err := s.activeResticRepositories(ctx, namespaces)
if err != nil {
return false, err
}
_, busy := active[repository]
return busy, nil
}
func uniqueSortedStrings(items []string) []string {
seen := map[string]struct{}{}
for _, item := range items {
item = strings.TrimSpace(item)
if item == "" {
continue
}
seen[item] = struct{}{}
}
out := make([]string, 0, len(seen))
for item := range seen {
out = append(out, item)
}
sort.Strings(out)
return out
}

7
internal/server/inventory_builder.go
View File

@ -31,6 +31,13 @@ func (s *Server) buildInventory(ctx context.Context) (api.InventoryResponse, err
AccessModes: summary.AccessModes,
Driver: s.cfg.BackupDriver,
}
if excluded, reason := s.pvcSummaryExcluded(summary); excluded {
entry.Healthy = true
entry.HealthReason = "excluded"
entry.Error = reason
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
continue
}
s.enrichPVCInventory(ctx, &entry, resticJobsByPVC, resticLookupErrors)
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
}

25
internal/server/inventory_builder_test.go
View File

@ -3,6 +3,7 @@ package server
import (
"context"
"errors"
"strings"
"testing"
"time"
@ -116,6 +117,30 @@ func TestBuildInventoryLonghornSortsNamespacesAndCalculatesHealth(t *testing.T)
}
}
func TestBuildInventoryMarksExcludedPVCsHealthy(t *testing.T) {
client := &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
},
},
}
srv := newInventoryTestServer(&config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ExcludedStorageClasses: []string{"local-path"},
}, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}})
inventory, err := srv.buildInventory(context.Background())
if err != nil {
t.Fatalf("build inventory: %v", err)
}
entry := inventory.Namespaces[0].PVCs[0]
if !entry.Healthy || entry.HealthReason != "excluded" || !strings.Contains(entry.Error, "local-path") {
t.Fatalf("expected excluded pvc to be treated as healthy exclusion, got %#v", entry)
}
}
func TestEnrichPVCInventoryCoversLonghornAndResticBranches(t *testing.T) {
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour)

88
internal/server/policy_cycle_test.go
View File

@ -93,7 +93,7 @@ func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) {
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/busy": {
{Name: "job-busy", Namespace: "apps", PVC: "busy", State: "Running", CreatedAt: recent},
{Name: "job-busy", Namespace: "apps", PVC: "busy", Repository: "s3:https://repo/root/isolated/apps/busy", State: "Running", CreatedAt: recent},
},
"apps/recent": {
{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
@ -193,3 +193,89 @@ func TestRunPolicyCycleHonorsBackupLimit(t *testing.T) {
t.Fatalf("expected cycle_limit metric, got %f", got)
}
}
func TestRunPolicyCycleSkipsExcludedPVCsAndBusyRepositories(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/other": {
{Name: "job-other", Namespace: "apps", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
},
},
},
},
}
srv := &Server{
cfg: &config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ResticRepository: "s3:https://repo/root",
ExcludedStorageClasses: []string{"local-path"},
},
client: client,
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
},
}
srv.runPolicyCycle(context.Background())
if len(client.backupRequests) != 0 {
t.Fatalf("expected no backup requests while shared repo is busy and cache is excluded, got %#v", client.backupRequests)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "repo_in_progress"}); got != 1 {
t.Fatalf("expected repo_in_progress metric, got %f", got)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "excluded"}); got != 1 {
t.Fatalf("expected excluded metric, got %f", got)
}
}
func TestRunPolicyCycleSkipsLiveExclusivePVCs(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}},
{Namespace: "apps", Name: "shared", VolumeName: "vol-shared", Phase: "Bound", AccessModes: []string{"ReadWriteMany"}},
},
pvcMounts: map[string][]k8s.PVCMount{
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
"apps/shared": {{PodName: "web-0", NodeName: "titan-07", Phase: "Running"}},
},
},
},
}
srv := &Server{
cfg: &config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ResticRepository: "s3:https://repo/root",
},
client: client,
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
},
}
srv.runPolicyCycle(context.Background())
if len(client.backupRequests) != 1 || client.backupRequests[0].PVC != "shared" {
t.Fatalf("expected only RWX pvc backup request, got %#v", client.backupRequests)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "live_rwo_mount"}); got != 1 {
t.Fatalf("expected live_rwo_mount skip metric, got %f", got)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
t.Fatalf("expected one successful policy backup, got %f", got)
}
}

36
internal/server/policy_runtime.go
View File

@ -10,6 +10,7 @@ import (
"time"
"scm.bstein.dev/bstein/soteria/internal/api"
"scm.bstein.dev/bstein/soteria/internal/k8s"
)
func (s *Server) runPolicyCycle(ctx context.Context) {
@ -35,7 +36,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
pvcMap := make(map[string]api.PVCInventory)
namespaceMap := make(map[string][]api.PVCInventory)
namespaceNames := make([]string, 0, len(inventory.Namespaces))
for _, group := range inventory.Namespaces {
namespaceNames = append(namespaceNames, group.Name)
for _, pvc := range group.PVCs {
key := pvc.Namespace + "/" + pvc.PVC
pvcMap[key] = pvc
@ -78,6 +81,17 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
}
sort.Strings(keys)
activeRepositories := map[string]struct{}{}
if s.cfg.BackupDriver == "restic" {
var err error
activeRepositories, err = s.activeResticRepositories(runCtx, namespaceNames)
if err != nil {
log.Printf("policy cycle active backup lookup failed: %v", err)
s.metrics.RecordPolicyBackup("active_lookup_error")
return
}
}
started := 0
for _, key := range keys {
effective := effectivePolicies[key]
@ -85,6 +99,20 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
if !ok {
continue
}
if excluded, _ := s.pvcExcluded(pvc.Namespace, pvc.PVC, pvc.StorageClass); excluded {
s.metrics.RecordPolicyBackup("excluded")
continue
}
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
if err != nil {
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
s.metrics.RecordPolicyBackup("active_lookup_error")
continue
}
if blocked {
s.metrics.RecordPolicyBackup("live_rwo_mount")
continue
}
// Never enqueue a new policy backup while one is already active for this PVC.
// This prevents runaway job storms when a backup is stuck Pending/Running.
if pvc.ActiveBackups > 0 {
@ -107,6 +135,11 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
s.metrics.RecordPolicyBackup("cycle_limit")
break
}
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, pvc.Namespace, pvc.PVC, effective.Dedupe)
if _, busy := activeRepositories[repository]; busy {
s.metrics.RecordPolicyBackup("repo_in_progress")
continue
}
_, result, err := s.executeBackup(runCtx, api.BackupRequest{
Namespace: pvc.Namespace,
@ -122,6 +155,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
log.Printf("policy backup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
continue
}
if s.cfg.BackupDriver == "restic" {
activeRepositories[repository] = struct{}{}
}
s.metrics.RecordPolicyBackup("success")
}
}

1
internal/server/server.go
View File

@ -24,6 +24,7 @@ type kubeClient interface {
ListBackupJobsForPVC(ctx context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error)
ReadBackupJobLog(ctx context.Context, namespace, jobName string) (string, error)
ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error)
ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]k8s.PVCMount, error)
PersistentVolumeClaimExists(ctx context.Context, namespace, pvcName string) (bool, error)
LoadSecretData(ctx context.Context, namespace, secretName, key string) ([]byte, error)
SaveSecretData(ctx context.Context, namespace, secretName, key string, value []byte, labels map[string]string) error

29
internal/server/server_test.go
View File

@ -21,6 +21,7 @@ import (
type fakeKubeClient struct {
pvcs []k8s.PVCSummary
pvcMounts map[string][]k8s.PVCMount
backupJobs map[string][]k8s.BackupJobSummary
jobLogs map[string]string
backupJobCalls int
@ -37,6 +38,23 @@ type secretErrorKubeClient struct {
}
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
for _, summary := range f.pvcs {
if summary.Namespace != namespace || summary.Name != pvcName {
continue
}
pvc := &corev1.PersistentVolumeClaim{}
pvc.Namespace = namespace
pvc.Name = pvcName
pvc.Spec.VolumeName = summary.VolumeName
for _, mode := range summary.AccessModes {
pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, corev1.PersistentVolumeAccessMode(mode))
}
if summary.StorageClass != "" {
storageClass := summary.StorageClass
pvc.Spec.StorageClassName = &storageClass
}
return summary.VolumeName, pvc, nil, nil
}
return namespace + "-" + pvcName + "-pv", nil, nil, nil
}
@ -70,6 +88,17 @@ func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, err
return f.pvcs, nil
}
func (f *fakeKubeClient) ListPVCMounts(_ context.Context, namespace, pvc string) ([]k8s.PVCMount, error) {
if f.pvcMounts == nil {
return nil, nil
}
key := namespace + "/" + pvc
items := f.pvcMounts[key]
out := make([]k8s.PVCMount, len(items))
copy(out, items)
return out, nil
}
func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
if f.backupJobs == nil {
return nil, nil

40
internal/server/ui-dist/assets/index-B48ZTq0Y.js Normal file
View File

File diff suppressed because one or more lines are too long

40
internal/server/ui-dist/assets/index-C8vHBL9g.js
View File

File diff suppressed because one or more lines are too long

2
internal/server/ui-dist/index.html
View File

@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Soteria Backup Console</title>
<script type="module" crossorigin src="/assets/index-C8vHBL9g.js"></script>
<script type="module" crossorigin src="/assets/index-B48ZTq0Y.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B24a4-XK.css">
</head>
<body>

2
internal/server/ui_renderer_test.go
View File

@ -94,7 +94,7 @@ func TestUIRendererServeAssetRejectsInvalidPathsAndServesRealAssets(t *testing.T
{name: "parent traversal", method: http.MethodGet, path: "/../secret.txt", ok: false},
{name: "trailing slash", method: http.MethodGet, path: "/assets/", ok: false, use: trailingSlashRenderer},
{name: "missing file", method: http.MethodGet, path: "/assets/does-not-exist.js", ok: false, use: missingPathRenderer},
{name: "real asset", method: http.MethodGet, path: "/assets/index-C8vHBL9g.js", ok: true},
{name: "real asset", method: http.MethodGet, path: "/assets/index-B48ZTq0Y.js", ok: true},
{name: "head asset", method: http.MethodHead, path: "/assets/index-B24a4-XK.css", ok: true},
}

7
web/src/components/PVCInventoryPanel.tsx
View File

@ -68,8 +68,9 @@ export function PVCInventoryPanel({
</div>
<div className="pvc-grid">
{namespace.pvcs.map((pvc) => {
const healthClass = pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad');
const healthLabel = pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason);
const isExcluded = pvc.health_reason === 'excluded';
const healthClass = isExcluded ? 'warn' : (pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad'));
const healthLabel = isExcluded ? 'Excluded' : (pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason));
const progressPct = Math.max(0, Math.min(100, Number(pvc.last_job_progress_pct || 0)));
const progressClass = progressChipClass(pvc.last_job_state);
const showProgress = Boolean(pvc.last_job_name) || (pvc.active_backups || 0) > 0;
@ -118,7 +119,7 @@ export function PVCInventoryPanel({
)}
{pvc.error && <p className="error tiny">{pvc.error}</p>}
<div className="actions">
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy}>
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy || isExcluded}>
Backup now
</button>
<button type="button" className="secondary" onClick={() => void onOpenPVCSelection(pvc.namespace, pvc.pvc)}>