backup: harden restic policy jobs
This commit is contained in:
parent
73525ad712
commit
fdab56ed82
@ -13,7 +13,8 @@ import (
|
|||||||
const (
|
const (
|
||||||
defaultBackupDriver = "longhorn"
|
defaultBackupDriver = "longhorn"
|
||||||
defaultResticImage = "restic/restic:0.16.4"
|
defaultResticImage = "restic/restic:0.16.4"
|
||||||
defaultJobTTLSeconds = 86400
|
defaultJobTTLSeconds = 259200
|
||||||
|
defaultJobBackoff = 1
|
||||||
defaultListenAddr = ":8080"
|
defaultListenAddr = ":8080"
|
||||||
defaultResticSecret = "soteria-restic"
|
defaultResticSecret = "soteria-restic"
|
||||||
defaultLonghornURL = "http://longhorn-backend.longhorn-system.svc:9500"
|
defaultLonghornURL = "http://longhorn-backend.longhorn-system.svc:9500"
|
||||||
@ -26,53 +27,59 @@ const (
|
|||||||
defaultUsageSecret = "soteria-backup-usage"
|
defaultUsageSecret = "soteria-backup-usage"
|
||||||
defaultB2ScanInterval = 15 * time.Minute
|
defaultB2ScanInterval = 15 * time.Minute
|
||||||
defaultB2ScanTimeout = 2 * time.Minute
|
defaultB2ScanTimeout = 2 * time.Minute
|
||||||
|
defaultExcludedSCs = "local-path"
|
||||||
serviceNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
|
serviceNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config holds the runtime settings used to build and serve Soteria.
|
// Config holds the runtime settings used to build and serve Soteria.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
SecretNamespace string
|
SecretNamespace string
|
||||||
BackupDriver string
|
BackupDriver string
|
||||||
ResticImage string
|
ResticImage string
|
||||||
ResticRepository string
|
ResticRepository string
|
||||||
ResticSecretName string
|
ResticSecretName string
|
||||||
ResticBackupArgs []string
|
ResticBackupArgs []string
|
||||||
ResticForgetArgs []string
|
ResticForgetArgs []string
|
||||||
S3Endpoint string
|
ResticPruneAfterBackup bool
|
||||||
S3Region string
|
ResticUnlockBeforeBackup bool
|
||||||
JobTTLSeconds int32
|
S3Endpoint string
|
||||||
JobNodeSelector map[string]string
|
S3Region string
|
||||||
JobCPURequest string
|
JobTTLSeconds int32
|
||||||
JobCPULimit string
|
JobBackoffLimit int32
|
||||||
JobMemoryRequest string
|
JobNodeSelector map[string]string
|
||||||
JobMemoryLimit string
|
JobCPURequest string
|
||||||
WorkerServiceAccount string
|
JobCPULimit string
|
||||||
ListenAddr string
|
JobMemoryRequest string
|
||||||
LonghornURL string
|
JobMemoryLimit string
|
||||||
LonghornBackupMode string
|
ExcludedStorageClasses []string
|
||||||
AuthRequired bool
|
ExcludedPVCs []string
|
||||||
AllowedGroups []string
|
WorkerServiceAccount string
|
||||||
AuthBearerTokens []string
|
ListenAddr string
|
||||||
MetricsRefreshInterval time.Duration
|
LonghornURL string
|
||||||
PolicyEvalInterval time.Duration
|
LonghornBackupMode string
|
||||||
PolicyBackupsPerCycle int
|
AuthRequired bool
|
||||||
PolicySecretName string
|
AllowedGroups []string
|
||||||
UsageSecretName string
|
AuthBearerTokens []string
|
||||||
BackupMaxAge time.Duration
|
MetricsRefreshInterval time.Duration
|
||||||
B2Enabled bool
|
PolicyEvalInterval time.Duration
|
||||||
B2Endpoint string
|
PolicyBackupsPerCycle int
|
||||||
B2Region string
|
PolicySecretName string
|
||||||
B2Buckets []string
|
UsageSecretName string
|
||||||
B2AccessKeyID string
|
BackupMaxAge time.Duration
|
||||||
B2SecretAccessKey string
|
B2Enabled bool
|
||||||
B2SecretNamespace string
|
B2Endpoint string
|
||||||
B2SecretName string
|
B2Region string
|
||||||
B2AccessKeyField string
|
B2Buckets []string
|
||||||
B2SecretKeyField string
|
B2AccessKeyID string
|
||||||
B2EndpointField string
|
B2SecretAccessKey string
|
||||||
B2ScanInterval time.Duration
|
B2SecretNamespace string
|
||||||
B2ScanTimeout time.Duration
|
B2SecretName string
|
||||||
|
B2AccessKeyField string
|
||||||
|
B2SecretKeyField string
|
||||||
|
B2EndpointField string
|
||||||
|
B2ScanInterval time.Duration
|
||||||
|
B2ScanTimeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load builds the runtime configuration from environment variables and cluster state.
|
// Load builds the runtime configuration from environment variables and cluster state.
|
||||||
@ -99,6 +106,8 @@ func Load() (*Config, error) {
|
|||||||
cfg.ResticSecretName = getenvDefault("SOTERIA_RESTIC_SECRET_NAME", defaultResticSecret)
|
cfg.ResticSecretName = getenvDefault("SOTERIA_RESTIC_SECRET_NAME", defaultResticSecret)
|
||||||
cfg.ResticBackupArgs = strings.Fields(getenv("SOTERIA_RESTIC_BACKUP_ARGS"))
|
cfg.ResticBackupArgs = strings.Fields(getenv("SOTERIA_RESTIC_BACKUP_ARGS"))
|
||||||
cfg.ResticForgetArgs = strings.Fields(getenv("SOTERIA_RESTIC_FORGET_ARGS"))
|
cfg.ResticForgetArgs = strings.Fields(getenv("SOTERIA_RESTIC_FORGET_ARGS"))
|
||||||
|
cfg.ResticPruneAfterBackup = getenvBool("SOTERIA_RESTIC_PRUNE_AFTER_BACKUP")
|
||||||
|
cfg.ResticUnlockBeforeBackup = getenvBool("SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP")
|
||||||
cfg.S3Endpoint = getenv("SOTERIA_S3_ENDPOINT")
|
cfg.S3Endpoint = getenv("SOTERIA_S3_ENDPOINT")
|
||||||
cfg.S3Region = getenv("SOTERIA_S3_REGION")
|
cfg.S3Region = getenv("SOTERIA_S3_REGION")
|
||||||
cfg.WorkerServiceAccount = getenv("SOTERIA_JOB_SERVICE_ACCOUNT")
|
cfg.WorkerServiceAccount = getenv("SOTERIA_JOB_SERVICE_ACCOUNT")
|
||||||
@ -108,6 +117,8 @@ func Load() (*Config, error) {
|
|||||||
cfg.JobCPULimit = getenv("SOTERIA_JOB_CPU_LIMIT")
|
cfg.JobCPULimit = getenv("SOTERIA_JOB_CPU_LIMIT")
|
||||||
cfg.JobMemoryRequest = getenv("SOTERIA_JOB_MEMORY_REQUEST")
|
cfg.JobMemoryRequest = getenv("SOTERIA_JOB_MEMORY_REQUEST")
|
||||||
cfg.JobMemoryLimit = getenv("SOTERIA_JOB_MEMORY_LIMIT")
|
cfg.JobMemoryLimit = getenv("SOTERIA_JOB_MEMORY_LIMIT")
|
||||||
|
cfg.ExcludedStorageClasses = parseCSV(getenvDefault("SOTERIA_EXCLUDED_STORAGE_CLASSES", defaultExcludedSCs))
|
||||||
|
cfg.ExcludedPVCs = parseCSV(getenv("SOTERIA_EXCLUDED_PVCS"))
|
||||||
cfg.LonghornURL = getenvDefault("SOTERIA_LONGHORN_URL", defaultLonghornURL)
|
cfg.LonghornURL = getenvDefault("SOTERIA_LONGHORN_URL", defaultLonghornURL)
|
||||||
cfg.LonghornBackupMode = getenvDefault("SOTERIA_LONGHORN_BACKUP_MODE", defaultLonghornMode)
|
cfg.LonghornBackupMode = getenvDefault("SOTERIA_LONGHORN_BACKUP_MODE", defaultLonghornMode)
|
||||||
cfg.AuthRequired = getenvBool("SOTERIA_AUTH_REQUIRED")
|
cfg.AuthRequired = getenvBool("SOTERIA_AUTH_REQUIRED")
|
||||||
@ -137,6 +148,11 @@ func Load() (*Config, error) {
|
|||||||
} else {
|
} else {
|
||||||
cfg.JobTTLSeconds = defaultJobTTLSeconds
|
cfg.JobTTLSeconds = defaultJobTTLSeconds
|
||||||
}
|
}
|
||||||
|
if retries, ok := getenvInt("SOTERIA_JOB_BACKOFF_LIMIT"); ok {
|
||||||
|
cfg.JobBackoffLimit = int32(retries)
|
||||||
|
} else {
|
||||||
|
cfg.JobBackoffLimit = defaultJobBackoff
|
||||||
|
}
|
||||||
|
|
||||||
if seconds, ok := getenvInt("SOTERIA_METRICS_REFRESH_SECONDS"); ok {
|
if seconds, ok := getenvInt("SOTERIA_METRICS_REFRESH_SECONDS"); ok {
|
||||||
cfg.MetricsRefreshInterval = time.Duration(seconds) * time.Second
|
cfg.MetricsRefreshInterval = time.Duration(seconds) * time.Second
|
||||||
@ -182,6 +198,12 @@ func Load() (*Config, error) {
|
|||||||
if cfg.JobNodeSelector == nil {
|
if cfg.JobNodeSelector == nil {
|
||||||
return nil, errors.New("SOTERIA_JOB_NODE_SELECTOR is invalid; expected key=value pairs")
|
return nil, errors.New("SOTERIA_JOB_NODE_SELECTOR is invalid; expected key=value pairs")
|
||||||
}
|
}
|
||||||
|
if cfg.JobTTLSeconds <= 0 {
|
||||||
|
return nil, errors.New("SOTERIA_JOB_TTL_SECONDS must be greater than zero")
|
||||||
|
}
|
||||||
|
if cfg.JobBackoffLimit < 0 {
|
||||||
|
return nil, errors.New("SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero")
|
||||||
|
}
|
||||||
if err := validateQuantity("SOTERIA_JOB_CPU_REQUEST", cfg.JobCPURequest); err != nil {
|
if err := validateQuantity("SOTERIA_JOB_CPU_REQUEST", cfg.JobCPURequest); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -22,13 +22,18 @@ func clearConfigEnv(t *testing.T) {
|
|||||||
"SOTERIA_RESTIC_SECRET_NAME",
|
"SOTERIA_RESTIC_SECRET_NAME",
|
||||||
"SOTERIA_RESTIC_BACKUP_ARGS",
|
"SOTERIA_RESTIC_BACKUP_ARGS",
|
||||||
"SOTERIA_RESTIC_FORGET_ARGS",
|
"SOTERIA_RESTIC_FORGET_ARGS",
|
||||||
|
"SOTERIA_RESTIC_PRUNE_AFTER_BACKUP",
|
||||||
|
"SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP",
|
||||||
"SOTERIA_S3_ENDPOINT",
|
"SOTERIA_S3_ENDPOINT",
|
||||||
"SOTERIA_S3_REGION",
|
"SOTERIA_S3_REGION",
|
||||||
"SOTERIA_JOB_SERVICE_ACCOUNT",
|
"SOTERIA_JOB_SERVICE_ACCOUNT",
|
||||||
"SOTERIA_LISTEN_ADDR",
|
"SOTERIA_LISTEN_ADDR",
|
||||||
"SOTERIA_JOB_NODE_SELECTOR",
|
"SOTERIA_JOB_NODE_SELECTOR",
|
||||||
|
"SOTERIA_JOB_BACKOFF_LIMIT",
|
||||||
"SOTERIA_LONGHORN_URL",
|
"SOTERIA_LONGHORN_URL",
|
||||||
"SOTERIA_LONGHORN_BACKUP_MODE",
|
"SOTERIA_LONGHORN_BACKUP_MODE",
|
||||||
|
"SOTERIA_EXCLUDED_STORAGE_CLASSES",
|
||||||
|
"SOTERIA_EXCLUDED_PVCS",
|
||||||
"SOTERIA_AUTH_REQUIRED",
|
"SOTERIA_AUTH_REQUIRED",
|
||||||
"SOTERIA_ALLOWED_GROUPS",
|
"SOTERIA_ALLOWED_GROUPS",
|
||||||
"SOTERIA_AUTH_BEARER_TOKENS",
|
"SOTERIA_AUTH_BEARER_TOKENS",
|
||||||
@ -82,6 +87,12 @@ func TestLoadDefaultsForLonghorn(t *testing.T) {
|
|||||||
if cfg.JobTTLSeconds != defaultJobTTLSeconds {
|
if cfg.JobTTLSeconds != defaultJobTTLSeconds {
|
||||||
t.Fatalf("expected default ttl %d, got %d", defaultJobTTLSeconds, cfg.JobTTLSeconds)
|
t.Fatalf("expected default ttl %d, got %d", defaultJobTTLSeconds, cfg.JobTTLSeconds)
|
||||||
}
|
}
|
||||||
|
if cfg.JobBackoffLimit != defaultJobBackoff {
|
||||||
|
t.Fatalf("expected default backoff %d, got %d", defaultJobBackoff, cfg.JobBackoffLimit)
|
||||||
|
}
|
||||||
|
if len(cfg.ExcludedStorageClasses) != 1 || cfg.ExcludedStorageClasses[0] != "local-path" {
|
||||||
|
t.Fatalf("expected local-path to be excluded by default, got %#v", cfg.ExcludedStorageClasses)
|
||||||
|
}
|
||||||
if cfg.MetricsRefreshInterval != defaultMetricsRefresh || cfg.PolicyEvalInterval != defaultPolicyEval {
|
if cfg.MetricsRefreshInterval != defaultMetricsRefresh || cfg.PolicyEvalInterval != defaultPolicyEval {
|
||||||
t.Fatalf("unexpected interval defaults: %#v", cfg)
|
t.Fatalf("unexpected interval defaults: %#v", cfg)
|
||||||
}
|
}
|
||||||
@ -108,6 +119,8 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
|
|||||||
withEnv(t, "SOTERIA_RESTIC_SECRET_NAME", "restic-creds")
|
withEnv(t, "SOTERIA_RESTIC_SECRET_NAME", "restic-creds")
|
||||||
withEnv(t, "SOTERIA_RESTIC_BACKUP_ARGS", "--one-file-system --exclude /tmp")
|
withEnv(t, "SOTERIA_RESTIC_BACKUP_ARGS", "--one-file-system --exclude /tmp")
|
||||||
withEnv(t, "SOTERIA_RESTIC_FORGET_ARGS", "--keep-last 5 --prune")
|
withEnv(t, "SOTERIA_RESTIC_FORGET_ARGS", "--keep-last 5 --prune")
|
||||||
|
withEnv(t, "SOTERIA_RESTIC_PRUNE_AFTER_BACKUP", "true")
|
||||||
|
withEnv(t, "SOTERIA_RESTIC_UNLOCK_BEFORE_BACKUP", "true")
|
||||||
withEnv(t, "SOTERIA_S3_ENDPOINT", "https://b2.example.invalid")
|
withEnv(t, "SOTERIA_S3_ENDPOINT", "https://b2.example.invalid")
|
||||||
withEnv(t, "SOTERIA_S3_REGION", "us-west-000")
|
withEnv(t, "SOTERIA_S3_REGION", "us-west-000")
|
||||||
withEnv(t, "SOTERIA_JOB_SERVICE_ACCOUNT", "soteria-worker")
|
withEnv(t, "SOTERIA_JOB_SERVICE_ACCOUNT", "soteria-worker")
|
||||||
@ -115,6 +128,9 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
|
|||||||
withEnv(t, "SOTERIA_JOB_CPU_LIMIT", "1")
|
withEnv(t, "SOTERIA_JOB_CPU_LIMIT", "1")
|
||||||
withEnv(t, "SOTERIA_JOB_MEMORY_REQUEST", "256Mi")
|
withEnv(t, "SOTERIA_JOB_MEMORY_REQUEST", "256Mi")
|
||||||
withEnv(t, "SOTERIA_JOB_MEMORY_LIMIT", "2Gi")
|
withEnv(t, "SOTERIA_JOB_MEMORY_LIMIT", "2Gi")
|
||||||
|
withEnv(t, "SOTERIA_JOB_BACKOFF_LIMIT", "2")
|
||||||
|
withEnv(t, "SOTERIA_EXCLUDED_STORAGE_CLASSES", "local-path, ephemeral")
|
||||||
|
withEnv(t, "SOTERIA_EXCLUDED_PVCS", "jenkins/jenkins-dind-cache, apps/tmp-*")
|
||||||
withEnv(t, "SOTERIA_LISTEN_ADDR", ":9090")
|
withEnv(t, "SOTERIA_LISTEN_ADDR", ":9090")
|
||||||
withEnv(t, "SOTERIA_JOB_NODE_SELECTOR", "hardware=rpi5,role=worker")
|
withEnv(t, "SOTERIA_JOB_NODE_SELECTOR", "hardware=rpi5,role=worker")
|
||||||
withEnv(t, "SOTERIA_AUTH_REQUIRED", "true")
|
withEnv(t, "SOTERIA_AUTH_REQUIRED", "true")
|
||||||
@ -156,12 +172,21 @@ func TestLoadSupportsResticAndB2Overrides(t *testing.T) {
|
|||||||
if strings.Join(cfg.ResticForgetArgs, " ") != "--keep-last 5 --prune" {
|
if strings.Join(cfg.ResticForgetArgs, " ") != "--keep-last 5 --prune" {
|
||||||
t.Fatalf("unexpected forget args: %#v", cfg.ResticForgetArgs)
|
t.Fatalf("unexpected forget args: %#v", cfg.ResticForgetArgs)
|
||||||
}
|
}
|
||||||
|
if !cfg.ResticPruneAfterBackup || !cfg.ResticUnlockBeforeBackup {
|
||||||
|
t.Fatalf("expected explicit restic safety overrides, got prune=%v unlock=%v", cfg.ResticPruneAfterBackup, cfg.ResticUnlockBeforeBackup)
|
||||||
|
}
|
||||||
if cfg.WorkerServiceAccount != "soteria-worker" || cfg.ListenAddr != ":9090" {
|
if cfg.WorkerServiceAccount != "soteria-worker" || cfg.ListenAddr != ":9090" {
|
||||||
t.Fatalf("unexpected worker/listen config: %#v", cfg)
|
t.Fatalf("unexpected worker/listen config: %#v", cfg)
|
||||||
}
|
}
|
||||||
if cfg.JobCPURequest != "100m" || cfg.JobCPULimit != "1" || cfg.JobMemoryRequest != "256Mi" || cfg.JobMemoryLimit != "2Gi" {
|
if cfg.JobCPURequest != "100m" || cfg.JobCPULimit != "1" || cfg.JobMemoryRequest != "256Mi" || cfg.JobMemoryLimit != "2Gi" {
|
||||||
t.Fatalf("unexpected job resources: %#v", cfg)
|
t.Fatalf("unexpected job resources: %#v", cfg)
|
||||||
}
|
}
|
||||||
|
if cfg.JobBackoffLimit != 2 {
|
||||||
|
t.Fatalf("unexpected job backoff: %#v", cfg.JobBackoffLimit)
|
||||||
|
}
|
||||||
|
if strings.Join(cfg.ExcludedStorageClasses, ",") != "local-path,ephemeral" || strings.Join(cfg.ExcludedPVCs, ",") != "jenkins/jenkins-dind-cache,apps/tmp-*" {
|
||||||
|
t.Fatalf("unexpected exclusions: storage=%#v pvcs=%#v", cfg.ExcludedStorageClasses, cfg.ExcludedPVCs)
|
||||||
|
}
|
||||||
if cfg.JobNodeSelector["hardware"] != "rpi5" || cfg.JobNodeSelector["role"] != "worker" {
|
if cfg.JobNodeSelector["hardware"] != "rpi5" || cfg.JobNodeSelector["role"] != "worker" {
|
||||||
t.Fatalf("unexpected node selector: %#v", cfg.JobNodeSelector)
|
t.Fatalf("unexpected node selector: %#v", cfg.JobNodeSelector)
|
||||||
}
|
}
|
||||||
@ -265,6 +290,20 @@ func TestLoadRejectsInvalidConfigurations(t *testing.T) {
|
|||||||
},
|
},
|
||||||
substr: "SOTERIA_BACKUP_MAX_AGE_HOURS must be greater than zero",
|
substr: "SOTERIA_BACKUP_MAX_AGE_HOURS must be greater than zero",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "invalid job ttl",
|
||||||
|
env: map[string]string{
|
||||||
|
"SOTERIA_JOB_TTL_SECONDS": "0",
|
||||||
|
},
|
||||||
|
substr: "SOTERIA_JOB_TTL_SECONDS must be greater than zero",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid job backoff",
|
||||||
|
env: map[string]string{
|
||||||
|
"SOTERIA_JOB_BACKOFF_LIMIT": "-1",
|
||||||
|
},
|
||||||
|
substr: "SOTERIA_JOB_BACKOFF_LIMIT must be greater than or equal to zero",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "b2 enabled without endpoint or secret",
|
name: "b2 enabled without endpoint or secret",
|
||||||
env: map[string]string{
|
env: map[string]string{
|
||||||
|
|||||||
@ -82,7 +82,7 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
|
|||||||
Dedupe: &dedupe,
|
Dedupe: &dedupe,
|
||||||
KeepLast: &keepLast,
|
KeepLast: &keepLast,
|
||||||
})
|
})
|
||||||
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
|
if strings.Contains(cmd, "restic unlock") || strings.Contains(cmd, "--prune") || !strings.Contains(cmd, "set -eu;") || !strings.Contains(cmd, "restic cat config") || !strings.Contains(cmd, "restic rebuild-index") || !strings.Contains(cmd, "restic init") || !strings.Contains(cmd, "--tag dedupe=off") || !strings.Contains(cmd, "--keep-last 3") {
|
||||||
t.Fatalf("expected backup command to include bootstrap/dedupe/keep-last logic, got %q", cmd)
|
t.Fatalf("expected backup command to include bootstrap/dedupe/keep-last logic, got %q", cmd)
|
||||||
}
|
}
|
||||||
if !strings.Contains(cmd, "--exclude *.tmp") || !strings.Contains(cmd, "--tag nightly") || !strings.Contains(cmd, "--tag prod") {
|
if !strings.Contains(cmd, "--exclude *.tmp") || !strings.Contains(cmd, "--tag nightly") || !strings.Contains(cmd, "--tag prod") {
|
||||||
@ -94,6 +94,17 @@ func TestJobNameBackupCommandAndResticEnvCoverRemainingBranches(t *testing.T) {
|
|||||||
t.Fatalf("expected backup command to use configured forget args, got %q", cmd)
|
t.Fatalf("expected backup command to use configured forget args, got %q", cmd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cfg.ResticPruneAfterBackup = true
|
||||||
|
cfg.ResticUnlockBeforeBackup = true
|
||||||
|
cfg.ResticForgetArgs = nil
|
||||||
|
cmd = backupCommand(cfg, api.BackupRequest{PVC: "data", KeepLast: &keepLast})
|
||||||
|
if !strings.Contains(cmd, "restic unlock") || !strings.Contains(cmd, "--keep-last 3 --prune") {
|
||||||
|
t.Fatalf("expected explicit unlock/prune opts to be honored, got %q", cmd)
|
||||||
|
}
|
||||||
|
if !strings.Contains(cmd, "ensure_restic_repo") || !strings.Contains(cmd, "already initialized") {
|
||||||
|
t.Fatalf("expected robust restic bootstrap handling, got %q", cmd)
|
||||||
|
}
|
||||||
|
|
||||||
env := resticEnv(cfg, "restic-secret", "")
|
env := resticEnv(cfg, "restic-secret", "")
|
||||||
values := map[string]string{}
|
values := map[string]string{}
|
||||||
for _, item := range env {
|
for _, item := range env {
|
||||||
|
|||||||
@ -80,7 +80,7 @@ func buildBackupJob(cfg *config.Config, req api.BackupRequest, jobName, secretNa
|
|||||||
Annotations: annotations,
|
Annotations: annotations,
|
||||||
},
|
},
|
||||||
Spec: batchv1.JobSpec{
|
Spec: batchv1.JobSpec{
|
||||||
BackoffLimit: int32Ptr(0),
|
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
|
||||||
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
|
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
|
||||||
Template: corev1.PodTemplateSpec{
|
Template: corev1.PodTemplateSpec{
|
||||||
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
|
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
|
||||||
@ -161,7 +161,7 @@ func buildRestoreJob(cfg *config.Config, req api.RestoreTestRequest, jobName, se
|
|||||||
Annotations: annotations,
|
Annotations: annotations,
|
||||||
},
|
},
|
||||||
Spec: batchv1.JobSpec{
|
Spec: batchv1.JobSpec{
|
||||||
BackoffLimit: int32Ptr(0),
|
BackoffLimit: int32Ptr(cfg.JobBackoffLimit),
|
||||||
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
|
TTLSecondsAfterFinished: int32Ptr(cfg.JobTTLSeconds),
|
||||||
Template: corev1.PodTemplateSpec{
|
Template: corev1.PodTemplateSpec{
|
||||||
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
|
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations},
|
||||||
@ -216,27 +216,38 @@ func backupCommand(cfg *config.Config, req api.BackupRequest) string {
|
|||||||
|
|
||||||
cmd := strings.Join(args, " ")
|
cmd := strings.Join(args, " ")
|
||||||
if keepLast > 0 {
|
if keepLast > 0 {
|
||||||
forget := strings.Join([]string{
|
forgetArgs := []string{
|
||||||
"restic", "forget",
|
"restic", "forget",
|
||||||
"--host", "soteria",
|
"--host", "soteria",
|
||||||
"--group-by", "host,tags",
|
"--group-by", "host,tags",
|
||||||
"--tag", fmt.Sprintf("pvc=%s", req.PVC),
|
"--tag", fmt.Sprintf("pvc=%s", req.PVC),
|
||||||
"--keep-last", strconv.Itoa(keepLast),
|
"--keep-last", strconv.Itoa(keepLast),
|
||||||
"--prune",
|
}
|
||||||
}, " ")
|
if cfg.ResticPruneAfterBackup {
|
||||||
|
forgetArgs = append(forgetArgs, "--prune")
|
||||||
|
}
|
||||||
|
forget := strings.Join(forgetArgs, " ")
|
||||||
cmd = fmt.Sprintf("%s && %s", cmd, forget)
|
cmd = fmt.Sprintf("%s && %s", cmd, forget)
|
||||||
} else if len(cfg.ResticForgetArgs) > 0 {
|
} else if len(cfg.ResticForgetArgs) > 0 {
|
||||||
forget := strings.Join(append([]string{"restic", "forget"}, cfg.ResticForgetArgs...), " ")
|
forget := strings.Join(append([]string{"restic", "forget"}, cfg.ResticForgetArgs...), " ")
|
||||||
cmd = fmt.Sprintf("%s && %s", cmd, forget)
|
cmd = fmt.Sprintf("%s && %s", cmd, forget)
|
||||||
}
|
}
|
||||||
|
|
||||||
bootstrap := "restic unlock >/dev/null 2>&1 || true; restic cat config >/dev/null 2>&1 || restic init; restic snapshots >/dev/null 2>&1 || restic rebuild-index"
|
bootstrapSteps := []string{resticRepoBootstrapCommand(), "ensure_restic_repo", "restic snapshots >/dev/null 2>&1 || restic rebuild-index"}
|
||||||
return "set -euo pipefail; " + bootstrap + "; " + cmd
|
if cfg.ResticUnlockBeforeBackup {
|
||||||
|
bootstrapSteps = append([]string{"restic unlock >/dev/null 2>&1 || true"}, bootstrapSteps...)
|
||||||
|
}
|
||||||
|
bootstrap := strings.Join(bootstrapSteps, "; ")
|
||||||
|
return "set -eu; " + bootstrap + "; " + cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func resticRepoBootstrapCommand() string {
|
||||||
|
return `ensure_restic_repo() { probe_log="$(mktemp)"; if restic cat config >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if restic init >"${probe_log}" 2>&1; then rm -f "${probe_log}"; return 0; fi; if grep -qi "already initialized" "${probe_log}"; then rm -f "${probe_log}"; return 0; fi; cat "${probe_log}" >&2; rm -f "${probe_log}"; return 1; }`
|
||||||
}
|
}
|
||||||
|
|
||||||
func restoreCommand(snapshot string) string {
|
func restoreCommand(snapshot string) string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"set -euo pipefail; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
|
"set -eu; rm -rf /cache/restore && mkdir -p /cache/restore; restic restore %s --target /cache/restore; if [ -d /cache/restore/data ]; then cp -a /cache/restore/data/. /restore/; else cp -a /cache/restore/. /restore/; fi",
|
||||||
snapshot,
|
snapshot,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,6 +19,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
|
|||||||
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
|
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
|
||||||
S3Region: "us-west-001",
|
S3Region: "us-west-001",
|
||||||
JobTTLSeconds: 3600,
|
JobTTLSeconds: 3600,
|
||||||
|
JobBackoffLimit: 1,
|
||||||
JobNodeSelector: map[string]string{"hardware": "rpi5"},
|
JobNodeSelector: map[string]string{"hardware": "rpi5"},
|
||||||
JobCPURequest: "100m",
|
JobCPURequest: "100m",
|
||||||
JobCPULimit: "1",
|
JobCPULimit: "1",
|
||||||
@ -55,7 +56,7 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
|
|||||||
if container.Name != "restic" || container.Image != "restic/restic:latest" {
|
if container.Name != "restic" || container.Image != "restic/restic:latest" {
|
||||||
t.Fatalf("expected restic container, got %#v", container)
|
t.Fatalf("expected restic container, got %#v", container)
|
||||||
}
|
}
|
||||||
if len(container.Args) != 1 || !strings.Contains(container.Args[0], "restic unlock") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
|
if len(container.Args) != 1 || strings.Contains(container.Args[0], "restic unlock") || strings.Contains(container.Args[0], "--prune") || !strings.Contains(container.Args[0], "restic backup /data") || !strings.Contains(container.Args[0], "--keep-last 3") {
|
||||||
t.Fatalf("expected backup command payload, got %#v", container.Args)
|
t.Fatalf("expected backup command payload, got %#v", container.Args)
|
||||||
}
|
}
|
||||||
if got := container.Resources.Requests[corev1.ResourceCPU]; got.String() != "100m" {
|
if got := container.Resources.Requests[corev1.ResourceCPU]; got.String() != "100m" {
|
||||||
@ -67,8 +68,8 @@ func TestBuildBackupJobAppliesSelectorsServiceAccountAndMetadata(t *testing.T) {
|
|||||||
if len(job.Spec.Template.Spec.Volumes) != 2 || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim == nil {
|
if len(job.Spec.Template.Spec.Volumes) != 2 || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim == nil {
|
||||||
t.Fatalf("expected pvc + cache volumes, got %#v", job.Spec.Template.Spec.Volumes)
|
t.Fatalf("expected pvc + cache volumes, got %#v", job.Spec.Template.Spec.Volumes)
|
||||||
}
|
}
|
||||||
if got := *job.Spec.BackoffLimit; got != 0 {
|
if got := *job.Spec.BackoffLimit; got != 1 {
|
||||||
t.Fatalf("expected zero backoff limit, got %d", got)
|
t.Fatalf("expected configured backoff limit, got %d", got)
|
||||||
}
|
}
|
||||||
if got := *job.Spec.TTLSecondsAfterFinished; got != 3600 {
|
if got := *job.Spec.TTLSecondsAfterFinished; got != 3600 {
|
||||||
t.Fatalf("expected ttl from config, got %d", got)
|
t.Fatalf("expected ttl from config, got %d", got)
|
||||||
@ -82,6 +83,7 @@ func TestBuildRestoreJobCoversDefaultAndTargetPVCRestoreShapes(t *testing.T) {
|
|||||||
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
|
S3Endpoint: "https://s3.us-west-001.backblazeb2.com",
|
||||||
S3Region: "us-west-001",
|
S3Region: "us-west-001",
|
||||||
JobTTLSeconds: 1800,
|
JobTTLSeconds: 1800,
|
||||||
|
JobBackoffLimit: 1,
|
||||||
JobNodeSelector: map[string]string{"hardware": "rpi5"},
|
JobNodeSelector: map[string]string{"hardware": "rpi5"},
|
||||||
WorkerServiceAccount: "soteria-worker",
|
WorkerServiceAccount: "soteria-worker",
|
||||||
}
|
}
|
||||||
|
|||||||
@ -109,6 +109,11 @@ func resticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string
|
|||||||
return appendRepositoryPath(base, suffix)
|
return appendRepositoryPath(base, suffix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResticRepositoryForBackup returns the repository path Soteria will use for a PVC backup.
|
||||||
|
func ResticRepositoryForBackup(base, namespace, pvc string, dedupe bool) string {
|
||||||
|
return resticRepositoryForBackup(base, namespace, pvc, dedupe)
|
||||||
|
}
|
||||||
|
|
||||||
func sanitizeRepositorySegment(value string) string {
|
func sanitizeRepositorySegment(value string) string {
|
||||||
sanitized := sanitizeName(value)
|
sanitized := sanitizeName(value)
|
||||||
if sanitized == "" {
|
if sanitized == "" {
|
||||||
|
|||||||
@ -199,6 +199,7 @@ func (c *Client) CreateBackupJob(ctx context.Context, cfg *config.Config, req ap
|
|||||||
job := buildBackupJob(cfg, req, jobName, secretName, repository, dedupeEnabled, keepLast)
|
job := buildBackupJob(cfg, req, jobName, secretName, repository, dedupeEnabled, keepLast)
|
||||||
if nodeName, err := c.resolvePVCMountedNode(ctx, req.Namespace, req.PVC); err == nil && nodeName != "" {
|
if nodeName, err := c.resolvePVCMountedNode(ctx, req.Namespace, req.PVC); err == nil && nodeName != "" {
|
||||||
job.Spec.Template.Spec.NodeName = nodeName
|
job.Spec.Template.Spec.NodeName = nodeName
|
||||||
|
job.Spec.Template.Spec.NodeSelector = nil
|
||||||
}
|
}
|
||||||
created, err := c.Clientset.BatchV1().Jobs(req.Namespace).Create(ctx, job, metav1.CreateOptions{})
|
created, err := c.Clientset.BatchV1().Jobs(req.Namespace).Create(ctx, job, metav1.CreateOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -190,6 +190,88 @@ func TestResolvePVCMountedNodeIgnoresDeadPodsAndFindsMountedClaim(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListPVCMountsMarksSoteriaAndIgnoresDeadPods(t *testing.T) {
|
||||||
|
now := metav1.NewTime(time.Now().UTC())
|
||||||
|
client := &Client{Clientset: k8sfake.NewSimpleClientset(
|
||||||
|
&corev1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "deleted", Namespace: "apps", DeletionTimestamp: &now},
|
||||||
|
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
|
||||||
|
Name: "data",
|
||||||
|
VolumeSource: corev1.VolumeSource{
|
||||||
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
Status: corev1.PodStatus{Phase: corev1.PodRunning},
|
||||||
|
},
|
||||||
|
&corev1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "failed", Namespace: "apps"},
|
||||||
|
Spec: corev1.PodSpec{Volumes: []corev1.Volume{{
|
||||||
|
Name: "data",
|
||||||
|
VolumeSource: corev1.VolumeSource{
|
||||||
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
Status: corev1.PodStatus{Phase: corev1.PodFailed},
|
||||||
|
},
|
||||||
|
&corev1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "apps"},
|
||||||
|
Spec: corev1.PodSpec{
|
||||||
|
NodeName: "titan-02",
|
||||||
|
Volumes: []corev1.Volume{{
|
||||||
|
Name: "data",
|
||||||
|
VolumeSource: corev1.VolumeSource{
|
||||||
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
Status: corev1.PodStatus{Phase: corev1.PodRunning},
|
||||||
|
},
|
||||||
|
&corev1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "soteria-backup-data",
|
||||||
|
Namespace: "apps",
|
||||||
|
Labels: map[string]string{
|
||||||
|
labelAppName: "soteria",
|
||||||
|
labelComponent: "backup",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Spec: corev1.PodSpec{
|
||||||
|
NodeName: "titan-02",
|
||||||
|
Volumes: []corev1.Volume{{
|
||||||
|
Name: "data",
|
||||||
|
VolumeSource: corev1.VolumeSource{
|
||||||
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
Status: corev1.PodStatus{Phase: corev1.PodRunning},
|
||||||
|
},
|
||||||
|
)}
|
||||||
|
|
||||||
|
mounts, err := client.ListPVCMounts(context.Background(), "apps", "data")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list pvc mounts: %v", err)
|
||||||
|
}
|
||||||
|
if len(mounts) != 2 {
|
||||||
|
t.Fatalf("expected app and soteria mounts only, got %#v", mounts)
|
||||||
|
}
|
||||||
|
if mounts[0].PodName != "app" || mounts[0].NodeName != "titan-02" || mounts[0].SoteriaBackup {
|
||||||
|
t.Fatalf("expected first mount to be the app pod, got %#v", mounts[0])
|
||||||
|
}
|
||||||
|
if mounts[1].PodName != "soteria-backup-data" || !mounts[1].SoteriaBackup {
|
||||||
|
t.Fatalf("expected second mount to be marked as soteria, got %#v", mounts[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
clientset := k8sfake.NewSimpleClientset()
|
||||||
|
clientset.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
|
||||||
|
return true, nil, errors.New("list mounts exploded")
|
||||||
|
})
|
||||||
|
client = &Client{Clientset: clientset}
|
||||||
|
if _, err := client.ListPVCMounts(context.Background(), "apps", "data"); err == nil || !strings.Contains(err.Error(), "list mounts exploded") {
|
||||||
|
t.Fatalf("expected list pvc mounts error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestReadBackupJobLogCoversSuccessAndListFailures(t *testing.T) {
|
func TestReadBackupJobLogCoversSuccessAndListFailures(t *testing.T) {
|
||||||
client := &Client{Clientset: k8sfake.NewSimpleClientset(
|
client := &Client{Clientset: k8sfake.NewSimpleClientset(
|
||||||
&corev1.Pod{
|
&corev1.Pod{
|
||||||
@ -301,6 +383,7 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
|
|||||||
ResticRepository: "s3:https://repo/root",
|
ResticRepository: "s3:https://repo/root",
|
||||||
ResticImage: "restic/restic:latest",
|
ResticImage: "restic/restic:latest",
|
||||||
JobTTLSeconds: 3600,
|
JobTTLSeconds: 3600,
|
||||||
|
JobNodeSelector: map[string]string{"kubernetes.io/arch": "arm64"},
|
||||||
WorkerServiceAccount: "soteria-sa",
|
WorkerServiceAccount: "soteria-sa",
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -345,6 +428,9 @@ func TestCreateBackupJobCoversValidationDryRunAndLiveCreation(t *testing.T) {
|
|||||||
if job.Spec.Template.Spec.NodeName != "titan-02" || job.Spec.Template.Spec.ServiceAccountName != "soteria-sa" {
|
if job.Spec.Template.Spec.NodeName != "titan-02" || job.Spec.Template.Spec.ServiceAccountName != "soteria-sa" {
|
||||||
t.Fatalf("expected node pin + service account, got %#v", job.Spec.Template.Spec)
|
t.Fatalf("expected node pin + service account, got %#v", job.Spec.Template.Spec)
|
||||||
}
|
}
|
||||||
|
if len(job.Spec.Template.Spec.NodeSelector) != 0 {
|
||||||
|
t.Fatalf("expected node selector to be cleared when backup is pinned to mounted PVC node, got %#v", job.Spec.Template.Spec.NodeSelector)
|
||||||
|
}
|
||||||
if job.Annotations[annotationDedupeEnabled] != "false" || job.Annotations[annotationKeepLast] != "3" {
|
if job.Annotations[annotationDedupeEnabled] != "false" || job.Annotations[annotationKeepLast] != "3" {
|
||||||
t.Fatalf("expected backup annotations, got %#v", job.Annotations)
|
t.Fatalf("expected backup annotations, got %#v", job.Annotations)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -21,6 +21,14 @@ type PVCSummary struct {
|
|||||||
AccessModes []string
|
AccessModes []string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PVCMount describes an active pod mounting a PVC.
|
||||||
|
type PVCMount struct {
|
||||||
|
PodName string
|
||||||
|
NodeName string
|
||||||
|
Phase string
|
||||||
|
SoteriaBackup bool
|
||||||
|
}
|
||||||
|
|
||||||
// ResolvePVCVolume loads a PVC and its bound PV so backup handlers can act on the real volume.
|
// ResolvePVCVolume loads a PVC and its bound PV so backup handlers can act on the real volume.
|
||||||
func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
|
func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
|
||||||
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
|
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
|
||||||
@ -39,6 +47,38 @@ func (c *Client) ResolvePVCVolume(ctx context.Context, namespace, pvcName string
|
|||||||
return pvc.Spec.VolumeName, pvc, pv, nil
|
return pvc.Spec.VolumeName, pvc, pv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListPVCMounts returns active pods in a namespace that reference the PVC.
|
||||||
|
func (c *Client) ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]PVCMount, error) {
|
||||||
|
pods, err := c.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list pods for pvc %s/%s: %w", namespace, pvcName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mounts := []PVCMount{}
|
||||||
|
for _, pod := range pods.Items {
|
||||||
|
if pod.DeletionTimestamp != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, volume := range pod.Spec.Volumes {
|
||||||
|
claim := volume.PersistentVolumeClaim
|
||||||
|
if claim == nil || claim.ClaimName != pvcName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mounts = append(mounts, PVCMount{
|
||||||
|
PodName: pod.Name,
|
||||||
|
NodeName: pod.Spec.NodeName,
|
||||||
|
Phase: string(pod.Status.Phase),
|
||||||
|
SoteriaBackup: pod.Labels[labelAppName] == "soteria" && pod.Labels[labelComponent] == "backup",
|
||||||
|
})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mounts, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ListBoundPVCs returns all bound PVCs in a stable namespace/name order.
|
// ListBoundPVCs returns all bound PVCs in a stable namespace/name order.
|
||||||
func (c *Client) ListBoundPVCs(ctx context.Context) ([]PVCSummary, error) {
|
func (c *Client) ListBoundPVCs(ctx context.Context) ([]PVCSummary, error) {
|
||||||
list, err := c.Clientset.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{})
|
list, err := c.Clientset.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{})
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package server
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
@ -238,6 +239,34 @@ func (s *Server) executeBackup(ctx context.Context, req api.BackupRequest, reque
|
|||||||
}
|
}
|
||||||
return response, "success", nil
|
return response, "success", nil
|
||||||
case "restic":
|
case "restic":
|
||||||
|
_, pvc, _, err := s.client.ResolvePVCVolume(ctx, req.Namespace, req.PVC)
|
||||||
|
if err != nil {
|
||||||
|
return api.BackupResponse{}, "validation_error", err
|
||||||
|
}
|
||||||
|
storageClass := ""
|
||||||
|
if pvc != nil && pvc.Spec.StorageClassName != nil {
|
||||||
|
storageClass = *pvc.Spec.StorageClassName
|
||||||
|
}
|
||||||
|
if excluded, reason := s.pvcExcluded(req.Namespace, req.PVC, storageClass); excluded {
|
||||||
|
return api.BackupResponse{}, "validation_error", errors.New(reason)
|
||||||
|
}
|
||||||
|
blocked, reason, err := s.liveExclusivePVCMounted(ctx, req.Namespace, req.PVC, pvcAccessModes(pvc))
|
||||||
|
if err != nil {
|
||||||
|
return api.BackupResponse{}, "backend_error", err
|
||||||
|
}
|
||||||
|
if blocked {
|
||||||
|
return api.BackupResponse{}, "validation_error", errors.New(reason)
|
||||||
|
}
|
||||||
|
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, req.Namespace, req.PVC, resolvedDedupe)
|
||||||
|
if !req.DryRun {
|
||||||
|
busy, err := s.resticRepositoryBusy(ctx, repository)
|
||||||
|
if err != nil {
|
||||||
|
return api.BackupResponse{}, "backend_error", err
|
||||||
|
}
|
||||||
|
if busy {
|
||||||
|
return api.BackupResponse{}, "in_progress", fmt.Errorf("restic repository already has an active Soteria backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
jobName, secretName, err := s.client.CreateBackupJob(ctx, s.cfg, req)
|
jobName, secretName, err := s.client.CreateBackupJob(ctx, s.cfg, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.BackupResponse{}, "backend_error", err
|
return api.BackupResponse{}, "backend_error", err
|
||||||
@ -268,9 +297,13 @@ func (s *Server) listNamespaceBoundPVCs(ctx context.Context, namespace string) (
|
|||||||
}
|
}
|
||||||
filtered := make([]k8s.PVCSummary, 0, len(items))
|
filtered := make([]k8s.PVCSummary, 0, len(items))
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if item.Namespace == namespace {
|
if item.Namespace != namespace {
|
||||||
filtered = append(filtered, item)
|
continue
|
||||||
}
|
}
|
||||||
|
if excluded, _ := s.pvcSummaryExcluded(item); excluded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
filtered = append(filtered, item)
|
||||||
}
|
}
|
||||||
return filtered, nil
|
return filtered, nil
|
||||||
}
|
}
|
||||||
@ -279,6 +312,8 @@ func backupStatusCode(result string) int {
|
|||||||
switch result {
|
switch result {
|
||||||
case "validation_error", "unsupported_driver":
|
case "validation_error", "unsupported_driver":
|
||||||
return http.StatusBadRequest
|
return http.StatusBadRequest
|
||||||
|
case "in_progress":
|
||||||
|
return http.StatusConflict
|
||||||
case "backend_error":
|
case "backend_error":
|
||||||
return http.StatusBadGateway
|
return http.StatusBadGateway
|
||||||
default:
|
default:
|
||||||
|
|||||||
@ -354,6 +354,69 @@ func TestHandleBackupAndNamespaceBackupValidationPaths(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandleBackupReturnsExecutionErrors(t *testing.T) {
|
||||||
|
srv := newBackupTestServer(
|
||||||
|
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
|
||||||
|
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
},
|
||||||
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
||||||
|
"ops/other": {
|
||||||
|
{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
|
||||||
|
)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/backup", strings.NewReader(`{"namespace":"apps","pvc":"data"}`))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
res := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(res, req)
|
||||||
|
|
||||||
|
assertErrorResponseContains(t, res, http.StatusConflict, "restic repository already has an active")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleNamespaceBackupReportsPartialFailureAndFiltersExcludedPVCs(t *testing.T) {
|
||||||
|
client := &policyCycleTestKubeClient{
|
||||||
|
inventoryTestKubeClient: &inventoryTestKubeClient{fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "alpha", VolumeName: "pv-alpha", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
{Namespace: "apps", Name: "beta", VolumeName: "pv-beta", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"},
|
||||||
|
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
createBackupErrForPVC: map[string]error{"beta": errors.New("create beta backup exploded")},
|
||||||
|
}
|
||||||
|
srv := newBackupTestServer(
|
||||||
|
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root", ExcludedStorageClasses: []string{"local-path"}},
|
||||||
|
client,
|
||||||
|
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
|
||||||
|
)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(`{"namespace":"apps","dedupe":false}`))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
res := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(res, req)
|
||||||
|
if res.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var payload api.NamespaceBackupResponse
|
||||||
|
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
|
||||||
|
t.Fatalf("decode payload: %v", err)
|
||||||
|
}
|
||||||
|
if payload.Total != 2 || payload.Succeeded != 1 || payload.Failed != 1 {
|
||||||
|
t.Fatalf("expected excluded cache to be filtered and beta failure reported, got %#v", payload)
|
||||||
|
}
|
||||||
|
if len(payload.Results) != 2 || payload.Results[0].PVC != "alpha" || payload.Results[1].PVC != "beta" || payload.Results[1].Status != "backend_error" {
|
||||||
|
t.Fatalf("unexpected namespace backup results: %#v", payload.Results)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteBackupAndStatusHelpers(t *testing.T) {
|
func TestExecuteBackupAndStatusHelpers(t *testing.T) {
|
||||||
t.Run("validation error", func(t *testing.T) {
|
t.Run("validation error", func(t *testing.T) {
|
||||||
srv := newBackupTestServer(
|
srv := newBackupTestServer(
|
||||||
@ -427,6 +490,57 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("restic excluded pvc", func(t *testing.T) {
|
||||||
|
srv := newBackupTestServer(
|
||||||
|
&config.Config{AuthRequired: false, BackupDriver: "restic", ExcludedStorageClasses: []string{"local-path"}},
|
||||||
|
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "cache", VolumeName: "pv-cache", Phase: "Bound", StorageClass: "local-path"}},
|
||||||
|
}}},
|
||||||
|
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
|
||||||
|
)
|
||||||
|
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "cache"}, "brad")
|
||||||
|
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "storage class local-path is excluded") {
|
||||||
|
t.Fatalf("expected excluded pvc validation error, got result=%q err=%v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("restic live rwo pvc", func(t *testing.T) {
|
||||||
|
srv := newBackupTestServer(
|
||||||
|
&config.Config{AuthRequired: false, BackupDriver: "restic"},
|
||||||
|
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}}},
|
||||||
|
pvcMounts: map[string][]k8s.PVCMount{
|
||||||
|
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
|
||||||
|
)
|
||||||
|
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
|
||||||
|
if result != "validation_error" || err == nil || !strings.Contains(err.Error(), "mounted by active pod gitea-0") {
|
||||||
|
t.Fatalf("expected live RWO validation error, got result=%q err=%v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("restic active repository conflict", func(t *testing.T) {
|
||||||
|
srv := newBackupTestServer(
|
||||||
|
&config.Config{AuthRequired: false, BackupDriver: "restic", ResticRepository: "s3:https://repo/root"},
|
||||||
|
&backupTestKubeClient{restoreTestKubeClient: &restoreTestKubeClient{fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "data", VolumeName: "pv-data", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
{Namespace: "ops", Name: "other", VolumeName: "pv-other", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
},
|
||||||
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
||||||
|
"ops/other": {{Name: "job-other", Namespace: "ops", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()}},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
&backupTestLonghornClient{restoreTestLonghornClient: &restoreTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}}},
|
||||||
|
)
|
||||||
|
_, result, err := srv.executeBackup(context.Background(), api.BackupRequest{Namespace: "apps", PVC: "data"}, "brad")
|
||||||
|
if result != "in_progress" || err == nil || !strings.Contains(err.Error(), "restic repository already has an active") {
|
||||||
|
t.Fatalf("expected active repository conflict, got result=%q err=%v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("unsupported driver", func(t *testing.T) {
|
t.Run("unsupported driver", func(t *testing.T) {
|
||||||
srv := newBackupTestServer(
|
srv := newBackupTestServer(
|
||||||
&config.Config{AuthRequired: false, BackupDriver: "mystery"},
|
&config.Config{AuthRequired: false, BackupDriver: "mystery"},
|
||||||
@ -443,6 +557,7 @@ func TestExecuteBackupAndStatusHelpers(t *testing.T) {
|
|||||||
testCases := map[string]int{
|
testCases := map[string]int{
|
||||||
"validation_error": http.StatusBadRequest,
|
"validation_error": http.StatusBadRequest,
|
||||||
"unsupported_driver": http.StatusBadRequest,
|
"unsupported_driver": http.StatusBadRequest,
|
||||||
|
"in_progress": http.StatusConflict,
|
||||||
"backend_error": http.StatusBadGateway,
|
"backend_error": http.StatusBadGateway,
|
||||||
"other": http.StatusInternalServerError,
|
"other": http.StatusInternalServerError,
|
||||||
}
|
}
|
||||||
|
|||||||
147
internal/server/backup_safety.go
Normal file
147
internal/server/backup_safety.go
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
||||||
|
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *Server) pvcSummaryExcluded(pvc k8s.PVCSummary) (bool, string) {
|
||||||
|
return s.pvcExcluded(pvc.Namespace, pvc.Name, pvc.StorageClass)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) pvcExcluded(namespace, pvc, storageClass string) (bool, string) {
|
||||||
|
storageClass = strings.TrimSpace(storageClass)
|
||||||
|
for _, excluded := range s.cfg.ExcludedStorageClasses {
|
||||||
|
if storageClass != "" && strings.EqualFold(storageClass, strings.TrimSpace(excluded)) {
|
||||||
|
return true, fmt.Sprintf("storage class %s is excluded from restic backups", storageClass)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.Trim(strings.TrimSpace(namespace)+"/"+strings.TrimSpace(pvc), "/")
|
||||||
|
for _, pattern := range s.cfg.ExcludedPVCs {
|
||||||
|
pattern = strings.TrimSpace(pattern)
|
||||||
|
if pattern == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if pattern == key {
|
||||||
|
return true, fmt.Sprintf("PVC %s is excluded from restic backups", key)
|
||||||
|
}
|
||||||
|
matched, err := path.Match(pattern, key)
|
||||||
|
if err == nil && matched {
|
||||||
|
return true, fmt.Sprintf("PVC %s matches excluded pattern %s", key, pattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) liveExclusivePVCMounted(ctx context.Context, namespace, pvc string, accessModes []string) (bool, string, error) {
|
||||||
|
if !hasExclusiveAccessMode(accessModes) {
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
mounts, err := s.client.ListPVCMounts(ctx, namespace, pvc)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
for _, mount := range mounts {
|
||||||
|
if mount.SoteriaBackup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
location := strings.TrimSpace(mount.PodName)
|
||||||
|
if mount.NodeName != "" {
|
||||||
|
location += " on " + mount.NodeName
|
||||||
|
}
|
||||||
|
return true, fmt.Sprintf("RWO PVC %s/%s is mounted by active pod %s; restic backups wait for the workload to release the claim or use Longhorn snapshots", namespace, pvc, location), nil
|
||||||
|
}
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasExclusiveAccessMode(accessModes []string) bool {
|
||||||
|
for _, mode := range accessModes {
|
||||||
|
switch corev1.PersistentVolumeAccessMode(strings.TrimSpace(mode)) {
|
||||||
|
case corev1.ReadWriteOnce, corev1.ReadWriteOncePod:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func pvcAccessModes(pvc *corev1.PersistentVolumeClaim) []string {
|
||||||
|
if pvc == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
modes := make([]string, 0, len(pvc.Spec.AccessModes))
|
||||||
|
for _, mode := range pvc.Spec.AccessModes {
|
||||||
|
modes = append(modes, string(mode))
|
||||||
|
}
|
||||||
|
return modes
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) activeResticRepositories(ctx context.Context, namespaces []string) (map[string]struct{}, error) {
|
||||||
|
active := map[string]struct{}{}
|
||||||
|
for _, namespace := range uniqueSortedStrings(namespaces) {
|
||||||
|
jobs, err := s.client.ListBackupJobs(ctx, namespace)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, job := range jobs {
|
||||||
|
if !backupJobInProgress(job.State) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
repository := strings.TrimSpace(job.Repository)
|
||||||
|
if repository == "" {
|
||||||
|
repository = strings.TrimSpace(s.cfg.ResticRepository)
|
||||||
|
}
|
||||||
|
if repository != "" {
|
||||||
|
active[repository] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return active, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) resticRepositoryBusy(ctx context.Context, repository string) (bool, error) {
|
||||||
|
repository = strings.TrimSpace(repository)
|
||||||
|
if repository == "" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
pvcs, err := s.client.ListBoundPVCs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
namespaces := make([]string, 0, len(pvcs))
|
||||||
|
for _, pvc := range pvcs {
|
||||||
|
namespaces = append(namespaces, pvc.Namespace)
|
||||||
|
}
|
||||||
|
active, err := s.activeResticRepositories(ctx, namespaces)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
_, busy := active[repository]
|
||||||
|
return busy, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueSortedStrings(items []string) []string {
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, item := range items {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[item] = struct{}{}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(seen))
|
||||||
|
for item := range seen {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@ -31,6 +31,13 @@ func (s *Server) buildInventory(ctx context.Context) (api.InventoryResponse, err
|
|||||||
AccessModes: summary.AccessModes,
|
AccessModes: summary.AccessModes,
|
||||||
Driver: s.cfg.BackupDriver,
|
Driver: s.cfg.BackupDriver,
|
||||||
}
|
}
|
||||||
|
if excluded, reason := s.pvcSummaryExcluded(summary); excluded {
|
||||||
|
entry.Healthy = true
|
||||||
|
entry.HealthReason = "excluded"
|
||||||
|
entry.Error = reason
|
||||||
|
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
|
||||||
|
continue
|
||||||
|
}
|
||||||
s.enrichPVCInventory(ctx, &entry, resticJobsByPVC, resticLookupErrors)
|
s.enrichPVCInventory(ctx, &entry, resticJobsByPVC, resticLookupErrors)
|
||||||
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
|
groups[summary.Namespace] = append(groups[summary.Namespace], entry)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package server
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -116,6 +117,30 @@ func TestBuildInventoryLonghornSortsNamespacesAndCalculatesHealth(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildInventoryMarksExcludedPVCsHealthy(t *testing.T) {
|
||||||
|
client := &inventoryTestKubeClient{
|
||||||
|
fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
srv := newInventoryTestServer(&config.Config{
|
||||||
|
BackupDriver: "restic",
|
||||||
|
BackupMaxAge: 24 * time.Hour,
|
||||||
|
ExcludedStorageClasses: []string{"local-path"},
|
||||||
|
}, client, &inventoryTestLonghornClient{fakeLonghornClient: &fakeLonghornClient{}})
|
||||||
|
|
||||||
|
inventory, err := srv.buildInventory(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build inventory: %v", err)
|
||||||
|
}
|
||||||
|
entry := inventory.Namespaces[0].PVCs[0]
|
||||||
|
if !entry.Healthy || entry.HealthReason != "excluded" || !strings.Contains(entry.Error, "local-path") {
|
||||||
|
t.Fatalf("expected excluded pvc to be treated as healthy exclusion, got %#v", entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEnrichPVCInventoryCoversLonghornAndResticBranches(t *testing.T) {
|
func TestEnrichPVCInventoryCoversLonghornAndResticBranches(t *testing.T) {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
recent := now.Add(-1 * time.Hour)
|
recent := now.Add(-1 * time.Hour)
|
||||||
|
|||||||
@ -93,7 +93,7 @@ func TestRunPolicyCycleCoversEffectivePoliciesAndResultTracking(t *testing.T) {
|
|||||||
},
|
},
|
||||||
backupJobs: map[string][]k8s.BackupJobSummary{
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
||||||
"apps/busy": {
|
"apps/busy": {
|
||||||
{Name: "job-busy", Namespace: "apps", PVC: "busy", State: "Running", CreatedAt: recent},
|
{Name: "job-busy", Namespace: "apps", PVC: "busy", Repository: "s3:https://repo/root/isolated/apps/busy", State: "Running", CreatedAt: recent},
|
||||||
},
|
},
|
||||||
"apps/recent": {
|
"apps/recent": {
|
||||||
{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
|
{Name: "job-recent", Namespace: "apps", PVC: "recent", State: "Completed", CreatedAt: recent, CompletionTime: recent, KeepLast: 1},
|
||||||
@ -193,3 +193,89 @@ func TestRunPolicyCycleHonorsBackupLimit(t *testing.T) {
|
|||||||
t.Fatalf("expected cycle_limit metric, got %f", got)
|
t.Fatalf("expected cycle_limit metric, got %f", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPolicyCycleSkipsExcludedPVCsAndBusyRepositories(t *testing.T) {
|
||||||
|
client := &policyCycleTestKubeClient{
|
||||||
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
||||||
|
fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", StorageClass: "astreae"},
|
||||||
|
{Namespace: "apps", Name: "cache", VolumeName: "vol-cache", Phase: "Bound", StorageClass: "local-path"},
|
||||||
|
},
|
||||||
|
backupJobs: map[string][]k8s.BackupJobSummary{
|
||||||
|
"apps/other": {
|
||||||
|
{Name: "job-other", Namespace: "apps", PVC: "other", Repository: "s3:https://repo/root", State: "Running", CreatedAt: time.Now().UTC()},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
srv := &Server{
|
||||||
|
cfg: &config.Config{
|
||||||
|
BackupDriver: "restic",
|
||||||
|
BackupMaxAge: 24 * time.Hour,
|
||||||
|
ResticRepository: "s3:https://repo/root",
|
||||||
|
ExcludedStorageClasses: []string{"local-path"},
|
||||||
|
},
|
||||||
|
client: client,
|
||||||
|
longhorn: &fakeLonghornClient{},
|
||||||
|
metrics: newTelemetry(),
|
||||||
|
policies: map[string]api.BackupPolicy{
|
||||||
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
srv.runPolicyCycle(context.Background())
|
||||||
|
|
||||||
|
if len(client.backupRequests) != 0 {
|
||||||
|
t.Fatalf("expected no backup requests while shared repo is busy and cache is excluded, got %#v", client.backupRequests)
|
||||||
|
}
|
||||||
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "repo_in_progress"}); got != 1 {
|
||||||
|
t.Fatalf("expected repo_in_progress metric, got %f", got)
|
||||||
|
}
|
||||||
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "excluded"}); got != 1 {
|
||||||
|
t.Fatalf("expected excluded metric, got %f", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPolicyCycleSkipsLiveExclusivePVCs(t *testing.T) {
|
||||||
|
client := &policyCycleTestKubeClient{
|
||||||
|
inventoryTestKubeClient: &inventoryTestKubeClient{
|
||||||
|
fakeKubeClient: &fakeKubeClient{
|
||||||
|
pvcs: []k8s.PVCSummary{
|
||||||
|
{Namespace: "apps", Name: "data", VolumeName: "vol-data", Phase: "Bound", AccessModes: []string{"ReadWriteOnce"}},
|
||||||
|
{Namespace: "apps", Name: "shared", VolumeName: "vol-shared", Phase: "Bound", AccessModes: []string{"ReadWriteMany"}},
|
||||||
|
},
|
||||||
|
pvcMounts: map[string][]k8s.PVCMount{
|
||||||
|
"apps/data": {{PodName: "gitea-0", NodeName: "titan-06", Phase: "Running"}},
|
||||||
|
"apps/shared": {{PodName: "web-0", NodeName: "titan-07", Phase: "Running"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
srv := &Server{
|
||||||
|
cfg: &config.Config{
|
||||||
|
BackupDriver: "restic",
|
||||||
|
BackupMaxAge: 24 * time.Hour,
|
||||||
|
ResticRepository: "s3:https://repo/root",
|
||||||
|
},
|
||||||
|
client: client,
|
||||||
|
longhorn: &fakeLonghornClient{},
|
||||||
|
metrics: newTelemetry(),
|
||||||
|
policies: map[string]api.BackupPolicy{
|
||||||
|
"apps__all": {ID: "apps__all", Namespace: "apps", IntervalHours: 1, Enabled: true, Dedupe: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
srv.runPolicyCycle(context.Background())
|
||||||
|
|
||||||
|
if len(client.backupRequests) != 1 || client.backupRequests[0].PVC != "shared" {
|
||||||
|
t.Fatalf("expected only RWX pvc backup request, got %#v", client.backupRequests)
|
||||||
|
}
|
||||||
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "live_rwo_mount"}); got != 1 {
|
||||||
|
t.Fatalf("expected live_rwo_mount skip metric, got %f", got)
|
||||||
|
}
|
||||||
|
if got := metricCount(srv.metrics.policyBackups, map[string]string{"result": "success"}); got != 1 {
|
||||||
|
t.Fatalf("expected one successful policy backup, got %f", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"scm.bstein.dev/bstein/soteria/internal/api"
|
"scm.bstein.dev/bstein/soteria/internal/api"
|
||||||
|
"scm.bstein.dev/bstein/soteria/internal/k8s"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *Server) runPolicyCycle(ctx context.Context) {
|
func (s *Server) runPolicyCycle(ctx context.Context) {
|
||||||
@ -35,7 +36,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
|
|
||||||
pvcMap := make(map[string]api.PVCInventory)
|
pvcMap := make(map[string]api.PVCInventory)
|
||||||
namespaceMap := make(map[string][]api.PVCInventory)
|
namespaceMap := make(map[string][]api.PVCInventory)
|
||||||
|
namespaceNames := make([]string, 0, len(inventory.Namespaces))
|
||||||
for _, group := range inventory.Namespaces {
|
for _, group := range inventory.Namespaces {
|
||||||
|
namespaceNames = append(namespaceNames, group.Name)
|
||||||
for _, pvc := range group.PVCs {
|
for _, pvc := range group.PVCs {
|
||||||
key := pvc.Namespace + "/" + pvc.PVC
|
key := pvc.Namespace + "/" + pvc.PVC
|
||||||
pvcMap[key] = pvc
|
pvcMap[key] = pvc
|
||||||
@ -78,6 +81,17 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
activeRepositories := map[string]struct{}{}
|
||||||
|
if s.cfg.BackupDriver == "restic" {
|
||||||
|
var err error
|
||||||
|
activeRepositories, err = s.activeResticRepositories(runCtx, namespaceNames)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("policy cycle active backup lookup failed: %v", err)
|
||||||
|
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
started := 0
|
started := 0
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
effective := effectivePolicies[key]
|
effective := effectivePolicies[key]
|
||||||
@ -85,6 +99,20 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if excluded, _ := s.pvcExcluded(pvc.Namespace, pvc.PVC, pvc.StorageClass); excluded {
|
||||||
|
s.metrics.RecordPolicyBackup("excluded")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
blocked, _, err := s.liveExclusivePVCMounted(runCtx, pvc.Namespace, pvc.PVC, pvc.AccessModes)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("policy cycle live PVC mount lookup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||||
|
s.metrics.RecordPolicyBackup("active_lookup_error")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if blocked {
|
||||||
|
s.metrics.RecordPolicyBackup("live_rwo_mount")
|
||||||
|
continue
|
||||||
|
}
|
||||||
// Never enqueue a new policy backup while one is already active for this PVC.
|
// Never enqueue a new policy backup while one is already active for this PVC.
|
||||||
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
// This prevents runaway job storms when a backup is stuck Pending/Running.
|
||||||
if pvc.ActiveBackups > 0 {
|
if pvc.ActiveBackups > 0 {
|
||||||
@ -107,6 +135,11 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
s.metrics.RecordPolicyBackup("cycle_limit")
|
s.metrics.RecordPolicyBackup("cycle_limit")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
repository := k8s.ResticRepositoryForBackup(s.cfg.ResticRepository, pvc.Namespace, pvc.PVC, effective.Dedupe)
|
||||||
|
if _, busy := activeRepositories[repository]; busy {
|
||||||
|
s.metrics.RecordPolicyBackup("repo_in_progress")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
_, result, err := s.executeBackup(runCtx, api.BackupRequest{
|
_, result, err := s.executeBackup(runCtx, api.BackupRequest{
|
||||||
Namespace: pvc.Namespace,
|
Namespace: pvc.Namespace,
|
||||||
@ -122,6 +155,9 @@ func (s *Server) runPolicyCycle(ctx context.Context) {
|
|||||||
log.Printf("policy backup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
log.Printf("policy backup failed for %s/%s: %v", pvc.Namespace, pvc.PVC, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if s.cfg.BackupDriver == "restic" {
|
||||||
|
activeRepositories[repository] = struct{}{}
|
||||||
|
}
|
||||||
s.metrics.RecordPolicyBackup("success")
|
s.metrics.RecordPolicyBackup("success")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -24,6 +24,7 @@ type kubeClient interface {
|
|||||||
ListBackupJobsForPVC(ctx context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error)
|
ListBackupJobsForPVC(ctx context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error)
|
||||||
ReadBackupJobLog(ctx context.Context, namespace, jobName string) (string, error)
|
ReadBackupJobLog(ctx context.Context, namespace, jobName string) (string, error)
|
||||||
ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error)
|
ListBoundPVCs(ctx context.Context) ([]k8s.PVCSummary, error)
|
||||||
|
ListPVCMounts(ctx context.Context, namespace, pvcName string) ([]k8s.PVCMount, error)
|
||||||
PersistentVolumeClaimExists(ctx context.Context, namespace, pvcName string) (bool, error)
|
PersistentVolumeClaimExists(ctx context.Context, namespace, pvcName string) (bool, error)
|
||||||
LoadSecretData(ctx context.Context, namespace, secretName, key string) ([]byte, error)
|
LoadSecretData(ctx context.Context, namespace, secretName, key string) ([]byte, error)
|
||||||
SaveSecretData(ctx context.Context, namespace, secretName, key string, value []byte, labels map[string]string) error
|
SaveSecretData(ctx context.Context, namespace, secretName, key string, value []byte, labels map[string]string) error
|
||||||
|
|||||||
@ -21,6 +21,7 @@ import (
|
|||||||
|
|
||||||
type fakeKubeClient struct {
|
type fakeKubeClient struct {
|
||||||
pvcs []k8s.PVCSummary
|
pvcs []k8s.PVCSummary
|
||||||
|
pvcMounts map[string][]k8s.PVCMount
|
||||||
backupJobs map[string][]k8s.BackupJobSummary
|
backupJobs map[string][]k8s.BackupJobSummary
|
||||||
jobLogs map[string]string
|
jobLogs map[string]string
|
||||||
backupJobCalls int
|
backupJobCalls int
|
||||||
@ -37,6 +38,23 @@ type secretErrorKubeClient struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
|
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
|
||||||
|
for _, summary := range f.pvcs {
|
||||||
|
if summary.Namespace != namespace || summary.Name != pvcName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pvc := &corev1.PersistentVolumeClaim{}
|
||||||
|
pvc.Namespace = namespace
|
||||||
|
pvc.Name = pvcName
|
||||||
|
pvc.Spec.VolumeName = summary.VolumeName
|
||||||
|
for _, mode := range summary.AccessModes {
|
||||||
|
pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, corev1.PersistentVolumeAccessMode(mode))
|
||||||
|
}
|
||||||
|
if summary.StorageClass != "" {
|
||||||
|
storageClass := summary.StorageClass
|
||||||
|
pvc.Spec.StorageClassName = &storageClass
|
||||||
|
}
|
||||||
|
return summary.VolumeName, pvc, nil, nil
|
||||||
|
}
|
||||||
return namespace + "-" + pvcName + "-pv", nil, nil, nil
|
return namespace + "-" + pvcName + "-pv", nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -70,6 +88,17 @@ func (f *fakeKubeClient) ListBoundPVCs(_ context.Context) ([]k8s.PVCSummary, err
|
|||||||
return f.pvcs, nil
|
return f.pvcs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeKubeClient) ListPVCMounts(_ context.Context, namespace, pvc string) ([]k8s.PVCMount, error) {
|
||||||
|
if f.pvcMounts == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
key := namespace + "/" + pvc
|
||||||
|
items := f.pvcMounts[key]
|
||||||
|
out := make([]k8s.PVCMount, len(items))
|
||||||
|
copy(out, items)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
|
func (f *fakeKubeClient) ListBackupJobsForPVC(_ context.Context, namespace, pvc string) ([]k8s.BackupJobSummary, error) {
|
||||||
if f.backupJobs == nil {
|
if f.backupJobs == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
40
internal/server/ui-dist/assets/index-B48ZTq0Y.js
Normal file
40
internal/server/ui-dist/assets/index-B48ZTq0Y.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>Soteria Backup Console</title>
|
<title>Soteria Backup Console</title>
|
||||||
<script type="module" crossorigin src="/assets/index-C8vHBL9g.js"></script>
|
<script type="module" crossorigin src="/assets/index-B48ZTq0Y.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-B24a4-XK.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-B24a4-XK.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@ -94,7 +94,7 @@ func TestUIRendererServeAssetRejectsInvalidPathsAndServesRealAssets(t *testing.T
|
|||||||
{name: "parent traversal", method: http.MethodGet, path: "/../secret.txt", ok: false},
|
{name: "parent traversal", method: http.MethodGet, path: "/../secret.txt", ok: false},
|
||||||
{name: "trailing slash", method: http.MethodGet, path: "/assets/", ok: false, use: trailingSlashRenderer},
|
{name: "trailing slash", method: http.MethodGet, path: "/assets/", ok: false, use: trailingSlashRenderer},
|
||||||
{name: "missing file", method: http.MethodGet, path: "/assets/does-not-exist.js", ok: false, use: missingPathRenderer},
|
{name: "missing file", method: http.MethodGet, path: "/assets/does-not-exist.js", ok: false, use: missingPathRenderer},
|
||||||
{name: "real asset", method: http.MethodGet, path: "/assets/index-C8vHBL9g.js", ok: true},
|
{name: "real asset", method: http.MethodGet, path: "/assets/index-B48ZTq0Y.js", ok: true},
|
||||||
{name: "head asset", method: http.MethodHead, path: "/assets/index-B24a4-XK.css", ok: true},
|
{name: "head asset", method: http.MethodHead, path: "/assets/index-B24a4-XK.css", ok: true},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -68,8 +68,9 @@ export function PVCInventoryPanel({
|
|||||||
</div>
|
</div>
|
||||||
<div className="pvc-grid">
|
<div className="pvc-grid">
|
||||||
{namespace.pvcs.map((pvc) => {
|
{namespace.pvcs.map((pvc) => {
|
||||||
const healthClass = pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad');
|
const isExcluded = pvc.health_reason === 'excluded';
|
||||||
const healthLabel = pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason);
|
const healthClass = isExcluded ? 'warn' : (pvc.healthy ? 'good' : (pvc.health_reason === 'in_progress' ? 'warn' : 'bad'));
|
||||||
|
const healthLabel = isExcluded ? 'Excluded' : (pvc.healthy ? 'Healthy' : formatLabel(pvc.health_reason));
|
||||||
const progressPct = Math.max(0, Math.min(100, Number(pvc.last_job_progress_pct || 0)));
|
const progressPct = Math.max(0, Math.min(100, Number(pvc.last_job_progress_pct || 0)));
|
||||||
const progressClass = progressChipClass(pvc.last_job_state);
|
const progressClass = progressChipClass(pvc.last_job_state);
|
||||||
const showProgress = Boolean(pvc.last_job_name) || (pvc.active_backups || 0) > 0;
|
const showProgress = Boolean(pvc.last_job_name) || (pvc.active_backups || 0) > 0;
|
||||||
@ -118,7 +119,7 @@ export function PVCInventoryPanel({
|
|||||||
)}
|
)}
|
||||||
{pvc.error && <p className="error tiny">{pvc.error}</p>}
|
{pvc.error && <p className="error tiny">{pvc.error}</p>}
|
||||||
<div className="actions">
|
<div className="actions">
|
||||||
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy}>
|
<button type="button" onClick={() => void onTriggerBackup(pvc.namespace, pvc.pvc)} disabled={busy || isExcluded}>
|
||||||
Backup now
|
Backup now
|
||||||
</button>
|
</button>
|
||||||
<button type="button" className="secondary" onClick={() => void onOpenPVCSelection(pvc.namespace, pvc.pvc)}>
|
<button type="button" className="secondary" onClick={() => void onOpenPVCSelection(pvc.namespace, pvc.pvc)}>
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user