bstein/soteria
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backup: harden restic policy jobs

Browse Source
This commit is contained in:
codex 2026-07-16 01:05:33 -03:00
parent 73525ad712
commit fdab56ed82
23 changed files with 802 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
internal/config/config.go
View File

@ -13,7 +13,8 @@ import (
const (
defaultBackupDriver = "longhorn"
defaultResticImage = "restic/restic:0.16.4"
defaultJobTTLSeconds = 86400
defaultJobTTLSeconds = 259200
defaultJobBackoff = 1
defaultListenAddr = ":8080"
defaultResticSecret = "soteria-restic"
defaultLonghornURL = "http://longhorn-backend.longhorn-system.svc:9500"
@ -26,53 +27,59 @@ const (
defaultUsageSecret = "soteria-backup-usage"
defaultB2ScanInterval = 15 * time.Minute
defaultB2ScanTimeout = 2 * time.Minute
defaultExcludedSCs = "local-path"
serviceNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
)
// Config holds the runtime settings used to build and serve Soteria.
type Config struct {
Namespace string
SecretNamespace string
BackupDriver string
ResticImage string
ResticRepository string
ResticSecretName string
ResticBackupArgs []string
ResticForgetArgs []string
S3Endpoint string
S3Region string
JobTTLSeconds int32
JobNodeSelector map[string]string
JobCPURequest string
JobCPULimit string
JobMemoryRequest string
JobMemoryLimit string
WorkerServiceAccount string
ListenAddr string
LonghornURL string
LonghornBackupMode string
AuthRequired bool
AllowedGroups []string
AuthBearerTokens []string
MetricsRefreshInterval time.Duration
PolicyEvalInterval time.Duration
PolicyBackupsPerCycle int
PolicySecretName string
UsageSecretName string
BackupMaxAge time.Duration
B2Enabled bool
B2Endpoint string
B2Region string
B2Buckets []string
B2AccessKeyID string
B2SecretAccessKey string
B2SecretNamespace string
B2SecretName string
B2AccessKeyField string
B2SecretKeyField string
B2EndpointField string
B2ScanInterval time.Duration
B2ScanTimeout time.Duration
Namespace string
SecretNamespace string
BackupDriver string
ResticImage string
ResticRepository string
ResticSecretName string
ResticBackupArgs []string
ResticForgetArgs []string
ResticPruneAfterBackup bool
ResticUnlockBeforeBackup bool
S3Endpoint string
S3Region string
JobTTLSeconds int32
JobBackoffLimit int32
JobNodeSelector map[string]string
JobCPURequest string
JobCPULimit string
JobMemoryRequest string
JobMemoryLimit string
ExcludedStorageClasses []string
ExcludedPVCs []string
WorkerServiceAccount string
ListenAddr string
LonghornURL string
LonghornBackupMode string
AuthRequired bool
AllowedGroups []string
AuthBearerTokens []string
MetricsRefreshInterval time.Duration
PolicyEvalInterval time.Duration
PolicyBackupsPerCycle int
PolicySecretName string
UsageSecretName string
BackupMaxAge time.Duration
B2Enabled bool
B2Endpoint string
B2Region string
B2Buckets []string
B2AccessKeyID string
B2SecretAccessKey string
B2SecretNamespace string
B2SecretName string
B2AccessKeyField string
B2SecretKeyField string
B2EndpointField string
B2ScanInterval time.Duration
B2ScanTimeout time.Duration
}
// Load builds the runtime configuration from environment variables and cluster state.
@ -99,6 +106,8 @@ func Load() (*Config, error) {
cfg.ResticSecretName = getenvDefault("SOTERIA_RESTIC_SECRET_NAME", defaultResticSecret)
cfg.ResticBackupArgs = strings.Fields(getenv("SOTERIA_RESTIC_BACKUP_ARGS"))
cfg.ResticForgetArgs = strings.Fields(getenv("SOTERIA_RESTIC_FORGET_ARGS"))
cfg.ResticPruneAfterBackup = getenvBool("SOTERIA_RESTIC_PRUNE_AFTER_BACKUP")
cfg.ResticUnlockBeforeBackup = getenvBool("SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP")
cfg.S3Endpoint = getenv("SOTERIA_S3_ENDPOINT")
cfg.S3Region = getenv("SOTERIA_S3_REGION")
cfg.WorkerServiceAccount = getenv("SOTERIA_JOB_SERVICE_ACCOUNT")
@ -108,6 +117,8 @@ func Load() (*Config, error) {
cfg.JobCPULimit = getenv("SOTERIA_JOB_CPU_LIMIT")
cfg.JobMemoryRequest = getenv("SOTERIA_JOB_MEMORY_REQUEST")
cfg.JobMemoryLimit = getenv("SOTERIA_JOB_MEMORY_LIMIT")
cfg.ExcludedStorageClasses = parseCSV(getenvDefault("SOTERIA_EXCLUDED_STORAGE_CLASSES", defaultExcludedSCs))
cfg.ExcludedPVCs = parseCSV(getenv("SOTERIA_EXCLUDED_PVCS"))
cfg.LonghornURL = getenvDefault("SOTERIA_LONGHORN_URL", defaultLonghornURL)
cfg.LonghornBackupMode = getenvDefault("SOTERIA_LONGHORN_BACKUP_MODE", defaultLonghornMode)
cfg.AuthRequired = getenvBool("SOTERIA_AUTH_REQUIRED")
@ -137,6 +148,11 @@ func Load() (*Config, error) {
} else {
cfg.JobTTLSeconds = defaultJobTTLSeconds
}
if retries, ok := getenvInt("SOTERIA_JOB_BACKOFF_LIMIT"); ok {
cfg.JobBackoffLimit = int32(retries)
} else {
cfg.JobBackoffLimit = defaultJobBackoff
}
if seconds, ok := getenvInt("SOTERIA_METRICS_REFRESH_SECONDS"); ok {
cfg.MetricsRefreshInterval = time.Duration(seconds) * time.Second
@ -182,6 +198,12 @@ func Load() (*Config, error) {
if cfg.JobNodeSelector == nil {
return nil, errors.New("SOTERIA_JOB_NODE_SELECTOR is invalid; expected key=value pairs")
}
if cfg.JobTTLSeconds <= 0 {
return nil, errors.New("SOTERIA_JOB_TTL_SECONDS must be greater than zero")
}
if cfg.JobBackoffLimit < 0 {
return nil, errors.New("SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero")
}
if err := validateQuantity("SOTERIA_JOB_CPU_REQUEST", cfg.JobCPURequest); err != nil {
return nil, err
}

39
internal/config/config_test.go
View File

@ -22,13 +22,18 @@ func clearConfigEnv(t *testing.T) {
"SOTERIA_RESTIC_SECRET_NAME",
"SOTERIA_RESTIC_BACKUP_ARGS",
"SOTERIA_RESTIC_FORGET_ARGS",
"SOTERIA_RESTIC_PRUNE_AFTER_BACKUP",
"SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP",
"SOTERIA_S3_ENDPOINT",
"SOTERIA_S3_REGION",
"SOTERIA_JOB_SERVICE_ACCOUNT",
"SOTERIA_LISTEN_ADDR",
"SOTERIA_JOB_NODE_SELECTOR",
"SOTERIA_JOB_BACKOFF_LIMIT",
"SOTERIA_LONGHORN_URL",
"SOTERIA_LONGHORN_BACKUP_MODE",
"SOTERIA_EXCLUDED_STORAGE_CLASSES",
"SOTERIA_EXCLUDED_PVCS",
"SOTERIA_AUTH_REQUIRED",
"SOTERIA_ALLOWED_GROUPS",
"SOTERIA_AUTH_BEARER_TOKENS",
@ -82,6 +87,12 @@ func TestLoadDefaultsForLonghorn(t *testing.T) {
if cfg.JobTTLSeconds != defaultJobTTLSeconds {
t.Fatalf("expected default ttl %d, got %d", defaultJobTTLSeconds, cfg.JobTTLSeconds)
}
if cfg.JobBackoffLimit != defaultJobBackoff {
t.Fatalf("expected default backoff %d, got %d", defaultJobBackoff, cfg.JobBackoffLimit)
}
if len(cfg.ExcludedStorageClasses) != 1 || cfg.ExcludedStorageClasses[0] != "local-path" {
t.Fatalf("expected local-path to be excluded by default, got %#v", cfg.ExcludedStorageClasses)
}
if cfg.MetricsRefreshInterval != defaultMetricsRefresh || cfg.PolicyEvalInterval != defaultPolicyEval {
t.Fatalf("unexpected interval defaults: %#v", cfg)
}
@ -108,6 +119,8 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
withEnv(t, "SOTERIA_RESTIC_SECRET_NAME", "restic-creds")
withEnv(t, "SOTERIA_RESTIC_BACKUP_ARGS", "--one-file-system --exclude /tmp")
withEnv(t, "SOTERIA_RESTIC_FORGET_ARGS", "--keep-last 5 --prune")
withEnv(t, "SOTERIA_RESTIC_PRUNE_AFTER_BACKUP", "true")
withEnv(t, "SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP", "true")
withEnv(t, "SOTERIA_S3_ENDPOINT", "https://b2.example.invalid")
withEnv(t, "SOTERIA_S3_REGION", "us-west-000")
withEnv(t, "SOTERIA_JOB_SERVICE_ACCOUNT", "soteria-worker")
@ -115,6 +128,9 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
withEnv(t, "SOTERIA_JOB_CPU_LIMIT", "1")
withEnv(t, "SOTERIA_JOB_MEMORY_REQUEST", "256Mi")
withEnv(t, "SOTERIA_JOB_MEMORY_LIMIT", "2Gi")
withEnv(t, "SOTERIA_JOB_BACKOFF_LIMIT", "2")
withEnv(t, "SOTERIA_EXCLUDED_STORAGE_CLASSES", "local-path, ephemeral")
withEnv(t, "SOTERIA_EXCLUDED_PVCS", "jenkins/jenkins-dind-cache, apps/tmp-*")
withEnv(t, "SOTERIA_LISTEN_ADDR", ":9090")
withEnv(t, "SOTERIA_JOB_NODE_SELECTOR", "hardware=rpi5,role=worker")
withEnv(t, "SOTERIA_AUTH_REQUIRED", "true")
@ -156,12 +172,21 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
if strings.Join(cfg.ResticForgetArgs, " ") != "--keep-last 5 --prune" {
t.Fatalf("unexpected forget args: %#v", cfg.ResticForgetArgs)
}
if !cfg.ResticPruneAfterBackup || !cfg.ResticUnlockBeforeBackup {
t.Fatalf("expected explicit restic safety overrides, got prune=%v unlock=%v", cfg.ResticPruneAfterBackup, cfg.ResticUnlockBeforeBackup)
}
if cfg.WorkerServiceAccount != "soteria-worker" || cfg.ListenAddr != ":9090" {
t.Fatalf("unexpected worker/listen config: %#v", cfg)
}
if cfg.JobCPURequest != "100m" || cfg.JobCPULimit != "1" || cfg.JobMemoryRequest != "256Mi" || cfg.JobMemoryLimit != "2Gi" {
t.Fatalf("unexpected job resources: %#v", cfg)
}
if cfg.JobBackoffLimit != 2 {
t.Fatalf("unexpected job backoff: %#v", cfg.JobBackoffLimit)
}
if strings.Join(cfg.ExcludedStorageClasses, ",") != "local-path,ephemeral" || strings.Join(cfg.ExcludedPVCs, ",") != "jenkins/jenkins-dind-cache,apps/tmp-*" {
t.Fatalf("unexpected exclusions: storage=%#v pvcs=%#v", cfg.ExcludedStorageClasses, cfg.ExcludedPVCs)
}
if cfg.JobNodeSelector["hardware"] != "rpi5" || cfg.JobNodeSelector["role"] != "worker" {
t.Fatalf("unexpected node selector: %#v", cfg.JobNodeSelector)
}
@ -265,6 +290,20 @@ func TestLoadRejectsInvalidConfigurations(t *testing.T) {
},
substr: "SOTERIA_BACKUP_MAX_AGE_HOURS must be greater than zero",
},
{
name: "invalid job ttl",
env: map[string]string{
"SOTERIA_JOB_TTL_SECONDS": "0",
},
substr: "SOTERIA_JOB_TTL_SECONDS must be greater than zero",
},
{
name: "invalid job backoff",
env: map[string]string{
"SOTERIA_JOB_BACKOFF_LIMIT": "-1",
},
substr: "SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero",
},
{
name: "b2 enabled without endpoint or secret",
env: map[string]string{

13
internal/k8s/job_helpers_test.go
View File

@ -82,7 +82,7 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
Dedupe: &dedupe,
KeepLast: &keepLast,
})
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
if strings.Contains(cmd, "restic unlock") || strings.Contains(cmd, "--prune") || !strings.Contains(cmd, "set -eu;") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
t.Fatalf("expected backup command to include bootstrap/dedupe/keep-last logic, got %q", cmd)
}
if !strings.Contains(cmd, "--exclude *.tmp") || !strings.Contains(cmd, "--tag nightly") || !strings.Contains(cmd, "--tag prod") {
@ -94,6 +94,17 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
t.Fatalf("expected backup command to use configured forget args, got %q", cmd)
}
cfg.ResticPruneAfterBackup = true
cfg.ResticUnlockBeforeBackup = true
cfg.ResticForgetArgs = nil
cmd = backupCommand(cfg, api.BackupRequest{PVC: "data", KeepLast: &keepLast})
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "--keep-last 3 --prune") {
t.Fatalf("expected explicit unlock/prune opts to be honored, got %q", cmd)
}
if !strings.Contains(cmd, "ensure_restic_repo") || !strings.Contains(cmd, "already initialized") {
t.Fatalf("expected robust restic bootstrap handling, got %q", cmd)
}
env := resticEnv(cfg, "restic-secret", "")
values := map[string]string{}
for _, item := range env {

27
internal/k8s/job_manifests.go
View File

@ -80,7 +80,7 @@ func buildBackupJob(cfg *config.Config, req api.BackupRequest, jobName, secretNa
Annotations: annotations,
},
Spec: batchv1.JobSpec{
BackoffLimit: int32Ptr(0),
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
@ -161,7 +161,7 @@ func buildRestoreJob(cfg *config.Config, req api.RestoreTestRequest, jobName, se
Annotations: annotations,
},
Spec: batchv1.JobSpec{
BackoffLimit: int32Ptr(0),
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
@ -216,27 +216,38 @@ func backupCommand(cfg *config.Config, req api.BackupRequest) string {
cmd := strings.Join(args, " ")
if keepLast > 0 {
forget := strings.Join([]string{
forgetArgs := []string{
"restic", "forget",
"--host", "soteria",
"--group-by", "host,tags",
"--tag", fmt.Sprintf("pvc=%s", req.PVC),
"--keep-last", strconv.Itoa(keepLast),
"--prune",
}, " ")
}
if cfg.ResticPruneAfterBackup {
forgetArgs = append(forgetArgs, "--prune")
}
forget := strings.Join(forgetArgs, " ")
cmd = fmt.Sprintf("%s && %s", cmd, forget)
} else if len(cfg.ResticForgetArgs) > 0 {
forget := strings.Join(append([]string{"restic", "forget"}, cfg.ResticForgetArgs...), " ")
cmd = fmt.Sprintf("%s && %s", cmd, forget)
}
bootstrap := "restic unlock >/dev/null 2>&1 || true; restic cat config >/dev/null 2>&1 || restic init; restic snapshots >/dev/null 2>&1 || restic rebuild-index"
return "set -euo pipefail; " + bootstrap + "; " + cmd
bootstrapSteps := []string{resticRepoBootstrapCommand(), "ensure_restic_repo", "restic snapshots >/dev/null 2>&1 || restic rebuild-index"}
if cfg.ResticUnlockBeforeBackup {
bootstrapSteps = append([]string{"restic unlock >/dev/null 2>&1 || true"}, bootstrapSteps...)
}
bootstrap := strings.Join(bootstrapSteps, "; ")
return "set -eu; " + bootstrap + "; " + cmd
}
func resticRepoBootstrapCommand() string {
return `ensure_restic_repo() { probe_log="$(mktemp)"; if restic cat config >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if restic init >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if grep -qi "already initialized" "${probe_log}"; then rm -f "${probe_log}"; return 0; fi; cat "${probe_log}" >&2; rm -f "${probe_log}"; return 1; }`
}
func restoreCommand(snapshot string) string {
return fmt.Sprintf(
"set -euo pipefail; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
"set -eu; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
snapshot,
)
}

8
internal/k8s/job_manifests_test.go
View File

@ -19,6 +19,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
S3Region: "us-west-001",
JobTTLSeconds: 3600,
JobBackoffLimit: 1,
JobNodeSelector: map[string]string{"hardware": "rpi5"},
JobCPURequest: "100m",
JobCPULimit: "1",
@ -55,7 +56,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
if container.Name != "restic" || container.Image != "restic/restic:latest" {
t.Fatalf("expected restic container, got %#v", container)
}
if len(container.Args) != 1 || !strings.Contains(container.Args[0], "restic unlock") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
if len(container.Args) != 1 || strings.Contains(container.Args[0], "restic unlock") || strings.Contains(container.Args[0], "--prune") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
t.Fatalf("expected backup command payload, got %#v", container.Args)
}
if got := container.Resources.Requests[corev1.ResourceCPU]; got.String() != "100m" {
@ -67,8 +68,8 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
if len(job.Spec.Template.Spec.Volumes) != 2 || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim == nil {
t.Fatalf("expected pvc + cache volumes, got %#v", job.Spec.Template.Spec.Volumes)
}
if got := *job.Spec.BackoffLimit; got != 0 {
t.Fatalf("expected zero backoff limit, got %d", got)
if got := *job.Spec.BackoffLimit; got != 1 {
t.Fatalf("expected configured backoff limit, got %d", got)
}
if got := *job.Spec.TTLSecondsAfterFinished; got != 3600 {
t.Fatalf("expected ttl from config, got %d", got)
@ -82,6 +83,7 @@ func TestBuildRestoreJobCoversDefaultAndTargetPVCRestoreShapes(t *testing.T) {
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
S3Region: "us-west-001",
JobTTLSeconds: 1800,
JobBackoffLimit: 1,
JobNodeSelector: map[string]string{"hardware": "rpi5"},
WorkerServiceAccount: "soteria-worker",
}

5
internal/k8s/job_support.go
View File

@ -109,6 +109,11 @@ func resticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string
return appendRepositoryPath(base, suffix)
}
// ResticRepositoryForBackup returns the repository path Soteria will use for a PVC backup.
func ResticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string {
return resticRepositoryForBackup(base, namespace, pvc, dedupe)
}
func sanitizeRepositorySegment(value string) string {
sanitized := sanitizeName(value)
if sanitized == "" {

1
internal/k8s/jobs.go
View File

@ -199,6 +199,7 @@ func (c *Client) CreateBackupJob(ctx context.Context, cfg *config.Config, req ap
job := buildBackupJob(cfg, req, jobName, secretName, repository, dedupeEnabled, keepLast)
if nodeName, err := c.resolvePVCMountedNode(ctx, req.Namespace, req.PVC); err == nil && nodeName != "" {
job.Spec.Template.Spec.NodeName = nodeName
job.Spec.Template.Spec.NodeSelector = nil
}
created, err := c.Clientset.BatchV1().Jobs(req.Namespace).Create(ctx, job, metav1.CreateOptions{})
if err != nil {

86
internal/k8s/jobs_test.go
View File

@ -190,6 +190,88 @@ func TestResolvePVCMountedNodeIgnoresDeadPodsAndFindsMountedClaim(t *testing.T)
}
}
func TestListPVCMountsMarksSoteriaAndIgnoresDeadPods(t *testing.T) {
now := metav1.NewTime(time.Now().UTC())
client := &Client{Clientset: k8sfake.NewSimpleClientset(
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "deleted", Namespace: "apps", DeletionTimestamp: &now},
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}}},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "failed", Namespace: "apps"},
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}}},
Status: corev1.PodStatus{Phase: corev1.PodFailed},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "apps"},
Spec: corev1.PodSpec{
NodeName: "titan-02",
Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "soteria-backup-data",
Namespace: "apps",
Labels: map[string]string{
labelAppName: "soteria",
labelComponent: "backup",
},
},
Spec: corev1.PodSpec{
NodeName: "titan-02",
Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
},
}},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
},
)}
mounts, err := client.ListPVCMounts(context.Background(), "apps", "data")
if err != nil {
t.Fatalf("list pvc mounts: %v", err)
}
if len(mounts) != 2 {
t.Fatalf("expected app and soteria mounts only, got %#v", mounts)
}
if mounts[0].PodName != "app" || mounts[0].NodeName != "titan-02" || mounts[0].SoteriaBackup {
t.Fatalf("expected first mount to be the app pod, got %#v", mounts[0])
}
if mounts[1].PodName != "soteria-backup-data" || !mounts[1].SoteriaBackup {
t.Fatalf("expected second mount to be marked as soteria, got %#v", mounts[1])
}
clientset := k8sfake.NewSimpleClientset()
clientset.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
return true, nil, errors.New("list mounts exploded")
})
client = &Client{Clientset: clientset}
if _, err := client.ListPVCMounts(context.Background(), "apps", "data"); err == nil || !strings.Contains(err.Error(), "list mounts exploded") {
t.Fatalf("expected list pvc mounts error, got %v", err)
}
}
func TestReadBackupJobLogCoversSuccessAndListFailures(t *testing.T) {
client := &Client{Clientset: k8sfake.NewSimpleClientset(
&corev1.Pod{
@ -301,6 +383,7 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
ResticRepository: "s3:https://repo/root",
ResticImage: "restic/restic:latest",
JobTTLSeconds: 3600,
JobNodeSelector: map[string]string{"kubernetes.io/arch": "arm64"},
WorkerServiceAccount: "soteria-sa",
}
@ -345,6 +428,9 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
if job.Spec.Template.Spec.NodeName != "titan-02" || job.Spec.Template.Spec.ServiceAccountName != "soteria-sa" {
t.Fatalf("expected node pin + service account, got %#v", job.Spec.Template.Spec)
}
if len(job.Spec.Template.Spec.NodeSelector) != 0 {
t.Fatalf("expected node selector to be cleared when backup is pinned to mounted PVC node, got %#v", job.Spec.Template.Spec.NodeSelector)
}
if job.Annotations[annotationDedupeEnabled] != "false" || job.Annotations[annotationKeepLast] != "3" {
t.Fatalf("expected backup annotations, got %#v", job.Annotations)
}

40
internal/k8s/volumes.go
View File

@ -21,6 +21,14 @@ type PVCSummary struct {
AccessModes []string
}
// PVCMount describes an active pod mounting a PVC.
type PVCMount struct {
PodName string
NodeName string
Phase string
SoteriaBackup bool
}
// ResolvePVCVolume loads a PVC and its bound PV so backup handlers can act on the real volume.
func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
@ -39,6 +47,38 @@ func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string
return pvc.Spec.VolumeName, pvc, pv, nil
}
// ListPVCMounts returns active pods in a namespace that reference the PVC.
func (c *Client) ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]PVCMount, error) {
pods, err := c.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("list pods for pvc %s/%s: %w", namespace, pvcName, err)
}
mounts := []PVCMount{}
for _, pod := range pods.Items {
if pod.DeletionTimestamp != nil {
continue
}
if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
continue
}
for _, volume := range pod.Spec.Volumes {
claim := volume.PersistentVolumeClaim
if claim == nil || claim.ClaimName != pvcName {
continue
}
mounts = append(mounts, PVCMount{
PodName: pod.Name,
NodeName: pod.Spec.NodeName,
Phase: string(pod.Status.Phase),
SoteriaBackup: pod.Labels[labelAppName] == "soteria" && pod.Labels[labelComponent] == "backup",
})
break
}
}
return mounts, nil
}
// ListBoundPVCs returns all bound PVCs in a stable namespace/name order.
func (c *Client) ListBoundPVCs(ctx context.Context) ([]PVCSummary, error) {
list, err := c.Clientset.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{})

39
internal/server/backup_handlers.go
View File

@ -3,6 +3,7 @@ package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
@ -238,6 +239,34 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
}
return response, "success", nil
case "restic":
_, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
if err != nil {
return api.BackupResponse{}, "validation_error", err
}
storageClass := ""
if pvc != nil && pvc.Spec.StorageClassName != nil {
storageClass = *pvc.Spec.StorageClassName
}
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
blocked, reason, err := s.liveExclusivePVCMounted(ctx, req.Namespace, req.PVC, pvcAccessModes(pvc))
if err != nil {
return api.BackupResponse{}, "backend_error", err
}
if blocked {
return api.BackupResponse{}, "validation_error", errors.New(reason)
}
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, req.Namespace, req.PVC, resolvedDedupe)
if !req.DryRun {
busy, err := s.resticRepositoryBusy(ctx, repository)
if err != nil {
return api.BackupResponse{}, "backend_error", err
}
if busy {
return api.BackupResponse{}, "in_progress", fmt.Errorf("restic repository already has an active Soteria backup")
}
}
jobName, secretName, err := s.client.CreateBackupJob(ctx, s.cfg, req)
if err != nil {
return api.BackupResponse{}, "backend_error", err
@ -268,9 +297,13 @@ func (s *Server) listNamespaceBoundPVCs(ctx context.Context, namespace string) (
}
filtered := make([]k8s.PVCSummary, 0, len(items))
for _, item := range items {
if item.Namespace == namespace {
filtered = append(filtered, item)
if item.Namespace != namespace {
continue
}
if excluded, _ := s.pvcSummaryExcluded(item); excluded {
continue
}
filtered = append(filtered, item)
}
return filtered, nil
}
@ -279,6 +312,8 @@ func backupStatusCode(result string) int {
switch result {
case "validation_error", "unsupported_driver":
return http.StatusBadRequest
case "in_progress":
return http.StatusConflict
case "backend_error":
return http.StatusBadGateway
default:

115
internal/server/backup_handlers_test.go
View File

@ -354,6 +354,69 @@ func TestHandleBackupAndNamespaceBackupValidationPaths(t *testing.T) {
})
}
func TestHandleBackupReturnsExecutionErrors(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"ops/other": {
{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(`{"namespace":"apps","pvc":"data"}`))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
assertErrorResponseContains(t, res, http.StatusConflict, "restic repository already has an active")
}
func TestHandleNamespaceBackupReportsPartialFailureAndFiltersExcludedPVCs(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "alpha", VolumeName: "pv-alpha", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "beta", VolumeName: "pv-beta", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
}},
createBackupErrForPVC: map[string]error{"beta": errors.New("create beta backup exploded")},
}
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root", ExcludedStorageClasses: []string{"local-path"}},
client,
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(`{"namespace":"apps","dedupe":false}`))
req.Header.Set("Content-Type", "application/json")
res := httptest.NewRecorder()
srv.Handler().ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
}
var payload api.NamespaceBackupResponse
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode payload: %v", err)
}
if payload.Total != 2 || payload.Succeeded != 1 || payload.Failed != 1 {
t.Fatalf("expected excluded cache to be filtered and beta failure reported, got %#v", payload)
}
if len(payload.Results) != 2 || payload.Results[0].PVC != "alpha" || payload.Results[1].PVC != "beta" || payload.Results[1].Status != "backend_error" {
t.Fatalf("unexpected namespace backup results: %#v", payload.Results)
}
}
func TestExecuteBackupAndStatusHelpers(t *testing.T) {
t.Run("validation error", func(t *testing.T) {
srv := newBackupTestServer(
@ -427,6 +490,57 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
}
})
t.Run("restic excluded pvc", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ExcludedStorageClasses: []string{"local-path"}},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"}},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "cache"}, "brad")
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "storage class local-path is excluded") {
t.Fatalf("expected excluded pvc validation error, got result=%q err=%v", result, err)
}
})
t.Run("restic live rwo pvc", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
pvcMounts: map[string][]k8s.PVCMount{
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "mounted by active pod gitea-0") {
t.Fatalf("expected live RWO validation error, got result=%q err=%v", result, err)
}
})
t.Run("restic active repository conflict", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"ops/other": {{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()}},
},
}}},
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
)
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
if result != "in_progress" || err == nil || !strings.Contains(err.Error(), "restic repository already has an active") {
t.Fatalf("expected active repository conflict, got result=%q err=%v", result, err)
}
})
t.Run("unsupported driver", func(t *testing.T) {
srv := newBackupTestServer(
&config.Config{AuthRequired: false, BackupDriver: "mystery"},
@ -443,6 +557,7 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
testCases := map[string]int{
"validation_error": http.StatusBadRequest,
"unsupported_driver": http.StatusBadRequest,
"in_progress": http.StatusConflict,
"backend_error": http.StatusBadGateway,
"other": http.StatusInternalServerError,
}

147
internal/server/backup_safety.go Normal file
View File

@ -0,0 +1,147 @@
package server
import (
"context"
"fmt"
"path"
"sort"
"strings"
"scm.bstein.dev/bstein/soteria/internal/k8s"
corev1 "k8s.io/api/core/v1"
)
func (s *Server) pvcSummaryExcluded(pvc k8s.PVCSummary) (bool, string) {
return s.pvcExcluded(pvc.Namespace, pvc.Name, pvc.StorageClass)
}
func (s *Server) pvcExcluded(namespace, pvc, storageClass string) (bool, string) {
storageClass = strings.TrimSpace(storageClass)
for _, excluded := range s.cfg.ExcludedStorageClasses {
if storageClass != "" && strings.EqualFold(storageClass, strings.TrimSpace(excluded)) {
return true, fmt.Sprintf("storage class %s is excluded from restic backups", storageClass)
}
}
key := strings.Trim(strings.TrimSpace(namespace)+"/"+strings.TrimSpace(pvc), "/")
for _, pattern := range s.cfg.ExcludedPVCs {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
continue
}
if pattern == key {
return true, fmt.Sprintf("PVC %s is excluded from restic backups", key)
}
matched, err := path.Match(pattern, key)
if err == nil && matched {
return true, fmt.Sprintf("PVC %s matches excluded pattern %s", key, pattern)
}
}
return false, ""
}
func (s *Server) liveExclusivePVCMounted(ctx context.Context, namespace, pvc string, accessModes []string) (bool, string, error) {
if !hasExclusiveAccessMode(accessModes) {
return false, "", nil
}
mounts, err := s.client.ListPVCMounts(ctx, namespace, pvc)
if err != nil {
return false, "", err
}
for _, mount := range mounts {
if mount.SoteriaBackup {
continue
}
location := strings.TrimSpace(mount.PodName)
if mount.NodeName != "" {
location += " on " + mount.NodeName
}
return true, fmt.Sprintf("RWO PVC %s/%s is mounted by active pod %s; restic backups wait for the workload to release the claim or use Longhorn snapshots", namespace, pvc, location), nil
}
return false, "", nil
}
func hasExclusiveAccessMode(accessModes []string) bool {
for _, mode := range accessModes {
switch corev1.PersistentVolumeAccessMode(strings.TrimSpace(mode)) {
case corev1.ReadWriteOnce, corev1.ReadWriteOncePod:
return true
}
}
return false
}
func pvcAccessModes(pvc *corev1.PersistentVolumeClaim) []string {
if pvc == nil {
return nil
}
modes := make([]string, 0, len(pvc.Spec.AccessModes))
for _, mode := range pvc.Spec.AccessModes {
modes = append(modes, string(mode))
}
return modes
}
func (s *Server) activeResticRepositories(ctx context.Context, namespaces []string) (map[string]struct{}, error) {
active := map[string]struct{}{}
for _, namespace := range uniqueSortedStrings(namespaces) {
jobs, err := s.client.ListBackupJobs(ctx, namespace)
if err != nil {
return nil, err
}
for _, job := range jobs {
if !backupJobInProgress(job.State) {
continue
}
repository := strings.TrimSpace(job.Repository)
if repository == "" {
repository = strings.TrimSpace(s.cfg.ResticRepository)
}
if repository != "" {
active[repository] = struct{}{}
}
}
}
return active, nil
}
func (s *Server) resticRepositoryBusy(ctx context.Context, repository string) (bool, error) {
repository = strings.TrimSpace(repository)
if repository == "" {
return false, nil
}
pvcs, err := s.client.ListBoundPVCs(ctx)
if err != nil {
return false, err
}
namespaces := make([]string, 0, len(pvcs))
for _, pvc := range pvcs {
namespaces = append(namespaces, pvc.Namespace)
}
active, err := s.activeResticRepositories(ctx, namespaces)
if err != nil {
return false, err
}
_, busy := active[repository]
return busy, nil
}
func uniqueSortedStrings(items []string) []string {
seen := map[string]struct{}{}
for _, item := range items {
item = strings.TrimSpace(item)
if item == "" {
continue
}
seen[item] = struct{}{}
}
out := make([]string, 0, len(seen))
for item := range seen {
out = append(out, item)
}
sort.Strings(out)
return out
}

7
internal/server/inventory_builder.go
View File

@ -31,6 +31,13 @@ func (s *Server) buildInventory(ctx context.Context) (api.InventoryResponse, err
AccessModes: summary.AccessModes,
Driver: s.cfg.BackupDriver,
}
if excluded, reason := s.pvcSummaryExcluded(summary); excluded {
entry.Healthy = true
entry.HealthReason = "excluded"
entry.Error = reason
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
continue
}
s.enrichPVCInventory(ctx, &entry, resticJobsByPVC, resticLookupErrors)
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
}

25
internal/server/inventory_builder_test.go
View File

@ -3,6 +3,7 @@ package server
import (
"context"
"errors"
"strings"
"testing"
"time"
@ -116,6 +117,30 @@ func TestBuildInventoryLonghornSortsNamespacesAndCalculatesHealth(t *testing.T)
}
}
func TestBuildInventoryMarksExcludedPVCsHealthy(t *testing.T) {
client := &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
},
},
}
srv := newInventoryTestServer(&config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ExcludedStorageClasses: []string{"local-path"},
}, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}})
inventory, err := srv.buildInventory(context.Background())
if err != nil {
t.Fatalf("build inventory: %v", err)
}
entry := inventory.Namespaces[0].PVCs[0]
if !entry.Healthy || entry.HealthReason != "excluded" || !strings.Contains(entry.Error, "local-path") {
t.Fatalf("expected excluded pvc to be treated as healthy exclusion, got %#v", entry)
}
}
func TestEnrichPVCInventoryCoversLonghornAndResticBranches(t *testing.T) {
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour)

88
internal/server/policy_cycle_test.go
View File

@ -93,7 +93,7 @@ func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) {
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/busy": {
{Name: "job-busy", Namespace: "apps", PVC: "busy", State: "Running", CreatedAt: recent},
{Name: "job-busy", Namespace: "apps", PVC: "busy", Repository: "s3:https://repo/root/isolated/apps/busy", State: "Running", CreatedAt: recent},
},
"apps/recent": {
{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
@ -193,3 +193,89 @@ func TestRunPolicyCycleHonorsBackupLimit(t *testing.T) {
t.Fatalf("expected cycle_limit metric, got %f", got)
}
}
func TestRunPolicyCycleSkipsExcludedPVCsAndBusyRepositories(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "astreae"},
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
},
backupJobs: map[string][]k8s.BackupJobSummary{
"apps/other": {
{Name: "job-other", Namespace: "apps", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
},
},
},
},
}
srv := &Server{
cfg: &config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ResticRepository: "s3:https://repo/root",
ExcludedStorageClasses: []string{"local-path"},
},
client: client,
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
},
}
srv.runPolicyCycle(context.Background())
if len(client.backupRequests) != 0 {
t.Fatalf("expected no backup requests while shared repo is busy and cache is excluded, got %#v", client.backupRequests)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "repo_in_progress"}); got != 1 {
t.Fatalf("expected repo_in_progress metric, got %f", got)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "excluded"}); got != 1 {
t.Fatalf("expected excluded metric, got %f", got)
}
}
func TestRunPolicyCycleSkipsLiveExclusivePVCs(t *testing.T) {
client := &policyCycleTestKubeClient{
inventoryTestKubeClient: &inventoryTestKubeClient{
fakeKubeClient: &fakeKubeClient{
pvcs: []k8s.PVCSummary{
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}},
{Namespace: "apps", Name: "shared", VolumeName: "vol-shared", Phase: "Bound", AccessModes: []string{"ReadWriteMany"}},
},
pvcMounts: map[string][]k8s.PVCMount{
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
"apps/shared": {{PodName: "web-0", NodeName: "titan-07", Phase: "Running"}},
},
},
},
}
srv := &Server{
cfg: &config.Config{
BackupDriver: "restic",
BackupMaxAge: 24 * time.Hour,
ResticRepository: "s3:https://repo/root",
},
client: client,
longhorn: &fakeLonghornClient{},
metrics: newTelemetry(),
policies: map[string]api.BackupPolicy{
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
},
}
srv.runPolicyCycle(context.Background())
if len(client.backupRequests) != 1 || client.backupRequests[0].PVC != "shared" {
t.Fatalf("expected only RWX pvc backup request, got %#v", client.backupRequests)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "live_rwo_mount"}); got != 1 {
t.Fatalf("expected live_rwo_mount skip metric, got %f", got)
}
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
t.Fatalf("expected one successful policy backup, got %f", got)
}
}

36
internal/server/policy_runtime.go
View File

@ -10,6 +10,7 @@ import (
"time"
"scm.bstein.dev/bstein/soteria/internal/api"
"scm.bstein.dev/bstein/soteria/internal/k8s"
)
func (s *Server) runPolicyCycle(ctx context.Context) {
@ -35,7 +36,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
pvcMap := make(map[string]api.PVCInventory)
namespaceMap := make(map[string][]api.PVCInventory)
namespaceNames := make([]string, 0, len(inventory.Namespaces))
for _, group := range inventory.Namespaces {
namespaceNames = append(namespaceNames, group.Name)
for _, pvc := range group.PVCs {
key := pvc.Namespace + "/" + pvc.PVC
pvcMap[key] = pvc
@ -78,6 +81,17 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
}
sort.Strings(keys)
activeRepositories := map[string]struct{}{}
if s.cfg.BackupDriver == "restic" {
var err error
activeRepositories, err = s.activeResticRepositories(runCtx, namespaceNames)
if err != nil {
log.Printf("policy cycle active backup lookup failed: %v", err)
s.metrics.RecordPolicyBackup("active_lookup_error")
return
}
}
started := 0
for _, key := range keys {
effective := effectivePolicies[key]
@ -85,6 +99,20 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
if !ok {
continue
}
if excluded, _ := s.pvcExcluded(pvc.Namespace, pvc.PVC, pvc.StorageClass); excluded {
s.metrics.RecordPolicyBackup("excluded")
continue
}
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
if err != nil {
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
s.metrics.RecordPolicyBackup("active_lookup_error")
continue
}
if blocked {
s.metrics.RecordPolicyBackup("live_rwo_mount")
continue
}
// Never enqueue a new policy backup while one is already active for this PVC.
// This prevents runaway job storms when a backup is stuck Pending/Running.
if pvc.ActiveBackups > 0 {
@ -107,6 +135,11 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
s.metrics.RecordPolicyBackup("cycle_limit")
break
}
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, pvc.Namespace, pvc.PVC, effective.Dedupe)
if _, busy := activeRepositories[repository]; busy {
s.metrics.RecordPolicyBackup("repo_in_progress")
continue
}
_, result, err := s.executeBackup(runCtx, api.BackupRequest{
Namespace: pvc.Namespace,
@ -122,6 +155,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
log.Printf("policy backup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
continue
}
if s.cfg.BackupDriver == "restic" {
activeRepositories[repository] = struct{}{}
}
s.metrics.RecordPolicyBackup("success")
}
}

1
internal/server/server.go
View File

@ -24,6 +24,7 @@ type kubeClient interface {
ListBackupJobsForPVC(ctx context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error)
ReadBackupJobLog(ctx context.Context, namespace, jobName string) (string, error)
ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error)
ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]k8s.PVCMount, error)
PersistentVolumeClaimExists(ctx context.Context, namespace, pvcName string) (bool, error)
LoadSecretData(ctx context.Context, namespace, secretName, key string) ([]byte, error)
SaveSecretData(ctx context.Context, namespace, secretName, key string, value []byte, labels map[string]string) error

29
internal/server/server_test.go
View File

@ -21,6 +21,7 @@ import (
type fakeKubeClient struct {
pvcs []k8s.PVCSummary
pvcMounts map[string][]k8s.PVCMount
backupJobs map[string][]k8s.BackupJobSummary
jobLogs map[string]string
backupJobCalls int
@ -37,6 +38,23 @@ type secretErrorKubeClient struct {
}
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
for _, summary := range f.pvcs {
if summary.Namespace != namespace || summary.Name != pvcName {
continue
}
pvc := &corev1.PersistentVolumeClaim{}
pvc.Namespace = namespace
pvc.Name = pvcName
pvc.Spec.VolumeName = summary.VolumeName
for _, mode := range summary.AccessModes {
pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, corev1.PersistentVolumeAccessMode(mode))
}
if summary.StorageClass != "" {
storageClass := summary.StorageClass
pvc.Spec.StorageClassName = &storageClass
}
return summary.VolumeName, pvc, nil, nil
}
return namespace + "-" + pvcName + "-pv", nil, nil, nil
}
@ -70,6 +88,17 @@ func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, err
return f.pvcs, nil
}
func (f *fakeKubeClient) ListPVCMounts(_ context.Context, namespace, pvc string) ([]k8s.PVCMount, error) {
if f.pvcMounts == nil {
return nil, nil
}
key := namespace + "/" + pvc
items := f.pvcMounts[key]
out := make([]k8s.PVCMount, len(items))
copy(out, items)
return out, nil
}
func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
if f.backupJobs == nil {
return nil, nil

40
internal/server/ui-dist/assets/index-B48ZTq0Y.js Normal file
View File

File diff suppressed because one or more lines are too long

40
internal/server/ui-dist/assets/index-C8vHBL9g.js
View File

File diff suppressed because one or more lines are too long

2
internal/server/ui-dist/index.html
View File

@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Soteria Backup Console</title>
<script type="module" crossorigin src="/assets/index-C8vHBL9g.js"></script>
<script type="module" crossorigin src="/assets/index-B48ZTq0Y.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B24a4-XK.css">
</head>
<body>

2
internal/server/ui_renderer_test.go
View File

@ -94,7 +94,7 @@ func TestUIRendererServeAssetRejectsInvalidPathsAndServesRealAssets(t *testing.T
{name: "parent traversal", method: http.MethodGet, path: "/../secret.txt", ok: false},
{name: "trailing slash", method: http.MethodGet, path: "/assets/", ok: false, use: trailingSlashRenderer},
{name: "missing file", method: http.MethodGet, path: "/assets/does-not-exist.js", ok: false, use: missingPathRenderer},
{name: "real asset", method: http.MethodGet, path: "/assets/index-C8vHBL9g.js", ok: true},
{name: "real asset", method: http.MethodGet, path: "/assets/index-B48ZTq0Y.js", ok: true},
{name: "head asset", method: http.MethodHead, path: "/assets/index-B24a4-XK.css", ok: true},
}

7
web/src/components/PVCInventoryPanel.tsx
View File

@ -68,8 +68,9 @@ export function PVCInventoryPanel({
</div>
<div className="pvc-grid">
{namespace.pvcs.map((pvc) => {
const healthClass = pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad');
const healthLabel = pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason);
const isExcluded = pvc.health_reason === 'excluded';
const healthClass = isExcluded ? 'warn' : (pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad'));
const healthLabel = isExcluded ? 'Excluded' : (pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason));
const progressPct = Math.max(0, Math.min(100, Number(pvc.last_job_progress_pct || 0)));
const progressClass = progressChipClass(pvc.last_job_state);
const showProgress = Boolean(pvc.last_job_name) || (pvc.active_backups || 0) > 0;
@ -118,7 +119,7 @@ export function PVCInventoryPanel({
)}
{pvc.error && <p className="error tiny">{pvc.error}</p>}
<div className="actions">
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy}>
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy || isExcluded}>
Backup now
</button>
<button type="button" className="secondary" onClick={() => void onOpenPVCSelection(pvc.namespace, pvc.pvc)}>