backup: add policy scheduler and namespace bulk operations
This commit is contained in:
parent
a9f97aab6b
commit
a5aa9e6a5f
55
README.md
55
README.md
@ -4,7 +4,9 @@ Soteria is an in-cluster service for PVC backup and restore operations. The curr
|
||||
|
||||
- Namespace-grouped PVC inventory for backup and restore selection.
|
||||
- On-demand backup creation for Longhorn volumes.
|
||||
- Namespace-wide backup and restore batch execution.
|
||||
- Restore into a new target PVC with conflict checks and best-effort cleanup on failure.
|
||||
- Policy-based scheduled backups (per PVC or all PVCs in a namespace), persisted in-cluster.
|
||||
- A simple built-in UI suitable for publishing behind an authenticated ingress.
|
||||
- Prometheus-format backup freshness telemetry for Grafana rollups.
|
||||
|
||||
@ -25,8 +27,13 @@ Protected endpoints when `SOTERIA_AUTH_REQUIRED=true`:
|
||||
- `GET /v1/inventory`
|
||||
- `GET /v1/backups?namespace=<ns>&pvc=<name>`
|
||||
- `POST /v1/backup`
|
||||
- `POST /v1/backup/namespace`
|
||||
- `POST /v1/restores`
|
||||
- `POST /v1/restores/namespace`
|
||||
- `POST /v1/restore-test` legacy alias for `/v1/restores`
|
||||
- `GET /v1/policies`
|
||||
- `POST /v1/policies`
|
||||
- `DELETE /v1/policies/<policy-id>`
|
||||
|
||||
## API examples
|
||||
|
||||
@ -114,6 +121,48 @@ Notes:
|
||||
- If Longhorn volume creation succeeds but PVC creation fails, Soteria attempts to delete the just-created restore volume.
|
||||
- You may provide `backup_url` directly instead of `snapshot`.
|
||||
|
||||
### POST /v1/backup/namespace
|
||||
|
||||
```json
|
||||
{
|
||||
"namespace": "ai",
|
||||
"dry_run": false
|
||||
}
|
||||
```
|
||||
|
||||
Runs backup for every currently bound PVC in the namespace and returns a per-PVC result list.
|
||||
|
||||
### POST /v1/restores/namespace
|
||||
|
||||
```json
|
||||
{
|
||||
"namespace": "ai",
|
||||
"target_namespace": "ai-restore",
|
||||
"target_prefix": "restore-20260412-",
|
||||
"snapshot": "",
|
||||
"dry_run": true
|
||||
}
|
||||
```
|
||||
|
||||
Runs restore planning/execution for every bound PVC in the source namespace. `snapshot` is optional and blank means latest completed backup per PVC.
|
||||
|
||||
### Policy API
|
||||
|
||||
Create or update a policy:
|
||||
|
||||
```json
|
||||
POST /v1/policies
|
||||
{
|
||||
"namespace": "ai",
|
||||
"pvc": "llm-cache",
|
||||
"interval_hours": 6,
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
- Leave `pvc` empty to target all PVCs in that namespace.
|
||||
- Policies are stored in secret `SOTERIA_POLICY_SECRET_NAME` under key `policies.json`.
|
||||
|
||||
## Authentication and authorization
|
||||
|
||||
When `SOTERIA_AUTH_REQUIRED=true`, Soteria expects trusted auth headers from a fronting proxy such as `oauth2-proxy`:
|
||||
@ -137,6 +186,9 @@ Implemented metrics:
|
||||
|
||||
- `soteria_backup_requests_total{driver,result}`
|
||||
- `soteria_restore_requests_total{driver,result}`
|
||||
- `soteria_policy_backups_total{result}`
|
||||
- `soteria_namespace_backup_requests_total{driver,result}`
|
||||
- `soteria_namespace_restore_requests_total{driver,result}`
|
||||
- `soteria_authz_denials_total{reason}`
|
||||
- `soteria_inventory_refresh_failures_total`
|
||||
- `soteria_inventory_refresh_timestamp_seconds`
|
||||
@ -171,6 +223,8 @@ Environment variables:
|
||||
- `SOTERIA_AUTH_BEARER_TOKENS` optional comma-separated bearer tokens
|
||||
- `SOTERIA_BACKUP_MAX_AGE_HOURS` default `24`
|
||||
- `SOTERIA_METRICS_REFRESH_SECONDS` default `300`
|
||||
- `SOTERIA_POLICY_EVAL_SECONDS` default `300`
|
||||
- `SOTERIA_POLICY_SECRET_NAME` default `soteria-policies`
|
||||
|
||||
## Secrets
|
||||
|
||||
@ -199,5 +253,6 @@ The example Service is annotated for Prometheus scraping of `/metrics`.
|
||||
## Notes
|
||||
|
||||
- Longhorn inventory and metrics are based on discovered backup records per PVC.
|
||||
- Scheduled policy execution currently applies to Longhorn driver.
|
||||
- Restic backup and restore execution exists, but inventory-style telemetry is currently Longhorn-focused.
|
||||
- For Atlas production, place Soteria behind an authenticated ingress and trust only proxy-injected auth headers.
|
||||
|
||||
@ -92,3 +92,80 @@ type AuthInfoResponse struct {
|
||||
Email string `json:"email,omitempty"`
|
||||
Groups []string `json:"groups,omitempty"`
|
||||
}
|
||||
|
||||
type BackupPolicy struct {
|
||||
ID string `json:"id"`
|
||||
Namespace string `json:"namespace"`
|
||||
PVC string `json:"pvc,omitempty"`
|
||||
IntervalHours float64 `json:"interval_hours"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
UpdatedAt string `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type BackupPolicyUpsertRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
PVC string `json:"pvc,omitempty"`
|
||||
IntervalHours float64 `json:"interval_hours"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
type BackupPolicyListResponse struct {
|
||||
Policies []BackupPolicy `json:"policies"`
|
||||
}
|
||||
|
||||
type NamespaceBackupRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
type NamespaceBackupResult struct {
|
||||
Namespace string `json:"namespace"`
|
||||
PVC string `json:"pvc"`
|
||||
Status string `json:"status"`
|
||||
Volume string `json:"volume,omitempty"`
|
||||
Backup string `json:"backup,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type NamespaceBackupResponse struct {
|
||||
Namespace string `json:"namespace"`
|
||||
RequestedBy string `json:"requested_by,omitempty"`
|
||||
Driver string `json:"driver"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Total int `json:"total"`
|
||||
Succeeded int `json:"succeeded"`
|
||||
Failed int `json:"failed"`
|
||||
Results []NamespaceBackupResult `json:"results"`
|
||||
}
|
||||
|
||||
type NamespaceRestoreRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
TargetNamespace string `json:"target_namespace,omitempty"`
|
||||
TargetPrefix string `json:"target_prefix,omitempty"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
type NamespaceRestoreResult struct {
|
||||
Namespace string `json:"namespace"`
|
||||
PVC string `json:"pvc"`
|
||||
TargetNamespace string `json:"target_namespace"`
|
||||
TargetPVC string `json:"target_pvc"`
|
||||
Status string `json:"status"`
|
||||
Volume string `json:"volume,omitempty"`
|
||||
BackupURL string `json:"backup_url,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type NamespaceRestoreResponse struct {
|
||||
Namespace string `json:"namespace"`
|
||||
TargetNamespace string `json:"target_namespace"`
|
||||
RequestedBy string `json:"requested_by,omitempty"`
|
||||
Driver string `json:"driver"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Total int `json:"total"`
|
||||
Succeeded int `json:"succeeded"`
|
||||
Failed int `json:"failed"`
|
||||
Results []NamespaceRestoreResult `json:"results"`
|
||||
}
|
||||
|
||||
@ -18,7 +18,9 @@ const (
|
||||
defaultLonghornMode = "incremental"
|
||||
defaultAllowedGroups = "admin,maintenance"
|
||||
defaultMetricsRefresh = 300 * time.Second
|
||||
defaultPolicyEval = 300 * time.Second
|
||||
defaultBackupMaxAge = 24 * time.Hour
|
||||
defaultPolicySecret = "soteria-policies"
|
||||
serviceNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
|
||||
)
|
||||
|
||||
@ -43,6 +45,8 @@ type Config struct {
|
||||
AllowedGroups []string
|
||||
AuthBearerTokens []string
|
||||
MetricsRefreshInterval time.Duration
|
||||
PolicyEvalInterval time.Duration
|
||||
PolicySecretName string
|
||||
BackupMaxAge time.Duration
|
||||
}
|
||||
|
||||
@ -80,7 +84,9 @@ func Load() (*Config, error) {
|
||||
cfg.AllowedGroups = parseCSV(getenvDefault("SOTERIA_ALLOWED_GROUPS", defaultAllowedGroups))
|
||||
cfg.AuthBearerTokens = parseCSV(getenv("SOTERIA_AUTH_BEARER_TOKENS"))
|
||||
cfg.MetricsRefreshInterval = defaultMetricsRefresh
|
||||
cfg.PolicyEvalInterval = defaultPolicyEval
|
||||
cfg.BackupMaxAge = defaultBackupMaxAge
|
||||
cfg.PolicySecretName = getenvDefault("SOTERIA_POLICY_SECRET_NAME", defaultPolicySecret)
|
||||
|
||||
if ttl, ok := getenvInt("SOTERIA_JOB_TTL_SECONDS"); ok {
|
||||
cfg.JobTTLSeconds = int32(ttl)
|
||||
@ -91,6 +97,9 @@ func Load() (*Config, error) {
|
||||
if seconds, ok := getenvInt("SOTERIA_METRICS_REFRESH_SECONDS"); ok {
|
||||
cfg.MetricsRefreshInterval = time.Duration(seconds) * time.Second
|
||||
}
|
||||
if seconds, ok := getenvInt("SOTERIA_POLICY_EVAL_SECONDS"); ok {
|
||||
cfg.PolicyEvalInterval = time.Duration(seconds) * time.Second
|
||||
}
|
||||
if hours, ok := getenvFloat("SOTERIA_BACKUP_MAX_AGE_HOURS"); ok {
|
||||
cfg.BackupMaxAge = time.Duration(hours * float64(time.Hour))
|
||||
}
|
||||
@ -130,6 +139,12 @@ func Load() (*Config, error) {
|
||||
if cfg.MetricsRefreshInterval <= 0 {
|
||||
return nil, errors.New("SOTERIA_METRICS_REFRESH_SECONDS must be greater than zero")
|
||||
}
|
||||
if cfg.PolicyEvalInterval <= 0 {
|
||||
return nil, errors.New("SOTERIA_POLICY_EVAL_SECONDS must be greater than zero")
|
||||
}
|
||||
if strings.TrimSpace(cfg.PolicySecretName) == "" {
|
||||
return nil, errors.New("SOTERIA_POLICY_SECRET_NAME must not be empty")
|
||||
}
|
||||
if cfg.BackupMaxAge <= 0 {
|
||||
return nil, errors.New("SOTERIA_BACKUP_MAX_AGE_HOURS must be greater than zero")
|
||||
}
|
||||
|
||||
73
internal/k8s/state.go
Normal file
73
internal/k8s/state.go
Normal file
@ -0,0 +1,73 @@
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func (c *Client) LoadSecretData(ctx context.Context, namespace, secretName, key string) ([]byte, error) {
|
||||
secret, err := c.Clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("get secret %s/%s: %w", namespace, secretName, err)
|
||||
}
|
||||
if secret.Data == nil {
|
||||
return nil, nil
|
||||
}
|
||||
value, ok := secret.Data[key]
|
||||
if !ok || len(value) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]byte, len(value))
|
||||
copy(out, value)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Client) SaveSecretData(ctx context.Context, namespace, secretName, key string, value []byte, labels map[string]string) error {
|
||||
secretClient := c.Clientset.CoreV1().Secrets(namespace)
|
||||
secret, err := secretClient.Get(ctx, secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("get secret %s/%s: %w", namespace, secretName, err)
|
||||
}
|
||||
secret = &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: secretName,
|
||||
Namespace: namespace,
|
||||
Labels: map[string]string{},
|
||||
},
|
||||
Type: corev1.SecretTypeOpaque,
|
||||
Data: map[string][]byte{},
|
||||
}
|
||||
}
|
||||
|
||||
if secret.Data == nil {
|
||||
secret.Data = map[string][]byte{}
|
||||
}
|
||||
secret.Data[key] = value
|
||||
|
||||
if secret.Labels == nil {
|
||||
secret.Labels = map[string]string{}
|
||||
}
|
||||
for labelKey, labelValue := range labels {
|
||||
secret.Labels[labelKey] = labelValue
|
||||
}
|
||||
|
||||
if secret.ResourceVersion == "" {
|
||||
if _, err := secretClient.Create(ctx, secret, metav1.CreateOptions{}); err != nil {
|
||||
return fmt.Errorf("create secret %s/%s: %w", namespace, secretName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := secretClient.Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("update secret %s/%s: %w", namespace, secretName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -20,6 +20,9 @@ type telemetry struct {
|
||||
mu sync.RWMutex
|
||||
backupRequests map[string]metricSample
|
||||
restoreRequests map[string]metricSample
|
||||
policyBackups map[string]metricSample
|
||||
namespaceBackupRequests map[string]metricSample
|
||||
namespaceRestoreReqs map[string]metricSample
|
||||
authzDenials map[string]metricSample
|
||||
inventoryRefreshFailure float64
|
||||
inventoryRefreshTime float64
|
||||
@ -31,13 +34,16 @@ type telemetry struct {
|
||||
|
||||
func newTelemetry() *telemetry {
|
||||
return &telemetry{
|
||||
backupRequests: map[string]metricSample{},
|
||||
restoreRequests: map[string]metricSample{},
|
||||
authzDenials: map[string]metricSample{},
|
||||
pvcBackupAgeHours: map[string]metricSample{},
|
||||
pvcBackupHealth: map[string]metricSample{},
|
||||
pvcBackupLastSuccess: map[string]metricSample{},
|
||||
pvcBackupCount: map[string]metricSample{},
|
||||
backupRequests: map[string]metricSample{},
|
||||
restoreRequests: map[string]metricSample{},
|
||||
policyBackups: map[string]metricSample{},
|
||||
namespaceBackupRequests: map[string]metricSample{},
|
||||
namespaceRestoreReqs: map[string]metricSample{},
|
||||
authzDenials: map[string]metricSample{},
|
||||
pvcBackupAgeHours: map[string]metricSample{},
|
||||
pvcBackupHealth: map[string]metricSample{},
|
||||
pvcBackupLastSuccess: map[string]metricSample{},
|
||||
pvcBackupCount: map[string]metricSample{},
|
||||
}
|
||||
}
|
||||
|
||||
@ -60,6 +66,24 @@ func (t *telemetry) RecordRestoreRequest(driver, result string) {
|
||||
incMetric(t.restoreRequests, map[string]string{"driver": driver, "result": result})
|
||||
}
|
||||
|
||||
func (t *telemetry) RecordPolicyBackup(result string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
incMetric(t.policyBackups, map[string]string{"result": result})
|
||||
}
|
||||
|
||||
func (t *telemetry) RecordNamespaceBackupRequest(driver, result string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
incMetric(t.namespaceBackupRequests, map[string]string{"driver": driver, "result": result})
|
||||
}
|
||||
|
||||
func (t *telemetry) RecordNamespaceRestoreRequest(driver, result string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
incMetric(t.namespaceRestoreReqs, map[string]string{"driver": driver, "result": result})
|
||||
}
|
||||
|
||||
func (t *telemetry) RecordAuthzDenied(reason string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
@ -115,6 +139,9 @@ func (t *telemetry) render() string {
|
||||
var b strings.Builder
|
||||
writeMetricFamily(&b, "soteria_backup_requests_total", "counter", "Backup requests handled by Soteria.", metricValues(t.backupRequests))
|
||||
writeMetricFamily(&b, "soteria_restore_requests_total", "counter", "Restore requests handled by Soteria.", metricValues(t.restoreRequests))
|
||||
writeMetricFamily(&b, "soteria_policy_backups_total", "counter", "Policy scheduler backup execution outcomes.", metricValues(t.policyBackups))
|
||||
writeMetricFamily(&b, "soteria_namespace_backup_requests_total", "counter", "Namespace-level backup request outcomes.", metricValues(t.namespaceBackupRequests))
|
||||
writeMetricFamily(&b, "soteria_namespace_restore_requests_total", "counter", "Namespace-level restore request outcomes.", metricValues(t.namespaceRestoreReqs))
|
||||
writeMetricFamily(&b, "soteria_authz_denials_total", "counter", "Authorization denials emitted by Soteria.", metricValues(t.authzDenials))
|
||||
writeMetricFamily(&b, "soteria_inventory_refresh_failures_total", "counter", "Inventory refresh failures while computing PVC backup telemetry.", []metricSample{{value: t.inventoryRefreshFailure}})
|
||||
writeMetricFamily(&b, "soteria_inventory_refresh_timestamp_seconds", "gauge", "Unix timestamp of the last successful inventory refresh.", []metricSample{{value: t.inventoryRefreshTime}})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@ -20,6 +21,7 @@ import (
|
||||
type fakeKubeClient struct {
|
||||
pvcs []k8s.PVCSummary
|
||||
targetExists bool
|
||||
secretData map[string][]byte
|
||||
}
|
||||
|
||||
func (f *fakeKubeClient) ResolvePVCVolume(_ context.Context, namespace, pvcName string) (string, *corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) {
|
||||
@ -42,6 +44,29 @@ func (f *fakeKubeClient) PersistentVolumeClaimExists(_ context.Context, _, _ str
|
||||
return f.targetExists, nil
|
||||
}
|
||||
|
||||
func (f *fakeKubeClient) LoadSecretData(_ context.Context, _, _, key string) ([]byte, error) {
|
||||
if f.secretData == nil {
|
||||
return nil, nil
|
||||
}
|
||||
value, ok := f.secretData[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
copyValue := make([]byte, len(value))
|
||||
copy(copyValue, value)
|
||||
return copyValue, nil
|
||||
}
|
||||
|
||||
func (f *fakeKubeClient) SaveSecretData(_ context.Context, _, _, key string, value []byte, _ map[string]string) error {
|
||||
if f.secretData == nil {
|
||||
f.secretData = map[string][]byte{}
|
||||
}
|
||||
copyValue := make([]byte, len(value))
|
||||
copy(copyValue, value)
|
||||
f.secretData[key] = copyValue
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeLonghornClient struct {
|
||||
backups []longhorn.Backup
|
||||
}
|
||||
@ -217,3 +242,129 @@ func TestMetricsStayPublic(t *testing.T) {
|
||||
t.Fatalf("expected prometheus metrics body, got %q", res.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoliciesCRUD(t *testing.T) {
|
||||
srv := &Server{
|
||||
cfg: &config.Config{AuthRequired: false, BackupDriver: "longhorn", Namespace: "maintenance", PolicySecretName: "soteria-policies"},
|
||||
client: &fakeKubeClient{},
|
||||
longhorn: &fakeLonghornClient{},
|
||||
metrics: newTelemetry(),
|
||||
policies: map[string]api.BackupPolicy{},
|
||||
}
|
||||
srv.handler = http.HandlerFunc(srv.route)
|
||||
|
||||
create := httptest.NewRequest(http.MethodPost, "/v1/policies", strings.NewReader(`{"namespace":"apps","interval_hours":6}`))
|
||||
create.Header.Set("Content-Type", "application/json")
|
||||
createRes := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(createRes, create)
|
||||
if createRes.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", createRes.Code, createRes.Body.String())
|
||||
}
|
||||
|
||||
var created api.BackupPolicy
|
||||
if err := json.Unmarshal(createRes.Body.Bytes(), &created); err != nil {
|
||||
t.Fatalf("decode policy: %v", err)
|
||||
}
|
||||
if created.Namespace != "apps" || created.IntervalHours != 6 {
|
||||
t.Fatalf("unexpected created policy: %#v", created)
|
||||
}
|
||||
|
||||
listReq := httptest.NewRequest(http.MethodGet, "/v1/policies", nil)
|
||||
listRes := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(listRes, listReq)
|
||||
if listRes.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", listRes.Code, listRes.Body.String())
|
||||
}
|
||||
var listPayload api.BackupPolicyListResponse
|
||||
if err := json.Unmarshal(listRes.Body.Bytes(), &listPayload); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
if len(listPayload.Policies) != 1 {
|
||||
t.Fatalf("expected one policy, got %#v", listPayload.Policies)
|
||||
}
|
||||
|
||||
deleteReq := httptest.NewRequest(http.MethodDelete, "/v1/policies/"+url.PathEscape(created.ID), nil)
|
||||
deleteRes := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(deleteRes, deleteReq)
|
||||
if deleteRes.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", deleteRes.Code, deleteRes.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceBackupDryRun(t *testing.T) {
|
||||
srv := &Server{
|
||||
cfg: &config.Config{
|
||||
AuthRequired: false,
|
||||
BackupDriver: "longhorn",
|
||||
},
|
||||
client: &fakeKubeClient{
|
||||
pvcs: []k8s.PVCSummary{
|
||||
{Namespace: "apps", Name: "data-a", VolumeName: "pv-apps-a", Phase: "Bound"},
|
||||
{Namespace: "apps", Name: "data-b", VolumeName: "pv-apps-b", Phase: "Bound"},
|
||||
{Namespace: "infra", Name: "data-c", VolumeName: "pv-infra-c", Phase: "Bound"},
|
||||
},
|
||||
},
|
||||
longhorn: &fakeLonghornClient{},
|
||||
metrics: newTelemetry(),
|
||||
policies: map[string]api.BackupPolicy{},
|
||||
}
|
||||
srv.handler = http.HandlerFunc(srv.route)
|
||||
|
||||
body := `{"namespace":"apps","dry_run":true}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/backup/namespace", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
res := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(res, req)
|
||||
|
||||
if res.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
|
||||
}
|
||||
var payload api.NamespaceBackupResponse
|
||||
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode payload: %v", err)
|
||||
}
|
||||
if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
|
||||
t.Fatalf("unexpected namespace backup payload: %#v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceRestoreDryRun(t *testing.T) {
|
||||
srv := &Server{
|
||||
cfg: &config.Config{
|
||||
AuthRequired: false,
|
||||
BackupDriver: "longhorn",
|
||||
},
|
||||
client: &fakeKubeClient{
|
||||
pvcs: []k8s.PVCSummary{
|
||||
{Namespace: "apps", Name: "cache", VolumeName: "pv-apps-cache", Phase: "Bound"},
|
||||
{Namespace: "apps", Name: "models", VolumeName: "pv-apps-models", Phase: "Bound"},
|
||||
},
|
||||
},
|
||||
longhorn: &fakeLonghornClient{},
|
||||
metrics: newTelemetry(),
|
||||
policies: map[string]api.BackupPolicy{},
|
||||
}
|
||||
srv.handler = http.HandlerFunc(srv.route)
|
||||
|
||||
body := `{"namespace":"apps","target_namespace":"restore","target_prefix":"drill","dry_run":true}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/restores/namespace", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
res := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(res, req)
|
||||
|
||||
if res.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", res.Code, res.Body.String())
|
||||
}
|
||||
var payload api.NamespaceRestoreResponse
|
||||
if err := json.Unmarshal(res.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode payload: %v", err)
|
||||
}
|
||||
if payload.Total != 2 || payload.Succeeded != 2 || payload.Failed != 0 {
|
||||
t.Fatalf("unexpected namespace restore payload: %#v", payload)
|
||||
}
|
||||
for _, result := range payload.Results {
|
||||
if !strings.HasPrefix(result.TargetPVC, "drill-") {
|
||||
t.Fatalf("expected target pvc with prefix drill-, got %q", result.TargetPVC)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -129,6 +129,16 @@
|
||||
}
|
||||
.muted { color: var(--muted); }
|
||||
.error { color: var(--bad); }
|
||||
.policy-item {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
padding: 10px 12px;
|
||||
background: rgba(255,255,255,0.65);
|
||||
}
|
||||
.mono {
|
||||
font-family: "SFMono-Regular", Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
||||
font-size: 0.88rem;
|
||||
}
|
||||
@media (max-width: 900px) {
|
||||
main { grid-template-columns: 1fr; }
|
||||
h1 { font-size: 1.7rem; }
|
||||
@ -167,6 +177,23 @@
|
||||
<h2>Last Action</h2>
|
||||
<pre id="result">No action yet.</pre>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Backup Policies</h2>
|
||||
<div class="stack" style="margin-bottom: 12px;">
|
||||
<label>Namespace<input id="policy-namespace" list="policy-namespace-options" placeholder="apps"></label>
|
||||
<datalist id="policy-namespace-options"></datalist>
|
||||
<label>PVC (optional, blank means all PVCs in namespace)<input id="policy-pvc" placeholder="cache-data"></label>
|
||||
<label>Interval hours<input id="policy-interval" type="number" min="1" step="1" value="24"></label>
|
||||
<label class="row" style="font-weight: 500;">
|
||||
<input id="policy-enabled" type="checkbox" checked style="width: auto;">
|
||||
Enabled
|
||||
</label>
|
||||
<button id="policy-save">Save policy</button>
|
||||
</div>
|
||||
<div id="policy-list" class="stack">
|
||||
<p class="muted">Loading policies...</p>
|
||||
</div>
|
||||
</section>
|
||||
</aside>
|
||||
</main>
|
||||
<script>
|
||||
@ -176,7 +203,15 @@
|
||||
const generatedAtEl = document.getElementById('generated-at');
|
||||
const authPillEl = document.getElementById('auth-pill');
|
||||
const refreshBtn = document.getElementById('refresh-btn');
|
||||
const policyListEl = document.getElementById('policy-list');
|
||||
const policyNamespaceEl = document.getElementById('policy-namespace');
|
||||
const policyNamespaceOptionsEl = document.getElementById('policy-namespace-options');
|
||||
const policyPVCEl = document.getElementById('policy-pvc');
|
||||
const policyIntervalEl = document.getElementById('policy-interval');
|
||||
const policyEnabledEl = document.getElementById('policy-enabled');
|
||||
const policySaveBtn = document.getElementById('policy-save');
|
||||
let latestInventory = null;
|
||||
let latestPolicies = [];
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value || '')
|
||||
@ -210,6 +245,19 @@
|
||||
return trimmed || 'restore-' + suffix;
|
||||
}
|
||||
|
||||
function suggestNamespaceRestorePrefix() {
|
||||
const now = new Date();
|
||||
const pad = (value) => String(value).padStart(2, '0');
|
||||
const suffix = [
|
||||
now.getUTCFullYear(),
|
||||
pad(now.getUTCMonth() + 1),
|
||||
pad(now.getUTCDate()),
|
||||
pad(now.getUTCHours()),
|
||||
pad(now.getUTCMinutes())
|
||||
].join('');
|
||||
return 'restore-' + suffix + '-';
|
||||
}
|
||||
|
||||
async function fetchJSON(url, options) {
|
||||
const response = await fetch(url, options);
|
||||
const text = await response.text();
|
||||
@ -250,6 +298,65 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerNamespaceBackup(namespace, dryRun) {
|
||||
try {
|
||||
const payload = await fetchJSON('/v1/backup/namespace', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ namespace, dry_run: dryRun })
|
||||
});
|
||||
showResult(payload);
|
||||
await loadInventory();
|
||||
} catch (error) {
|
||||
showResult({ error: error.message, namespace, dry_run: dryRun });
|
||||
}
|
||||
}
|
||||
|
||||
async function showNamespaceRestore(namespace) {
|
||||
const namespaceOptions = (latestInventory && latestInventory.namespaces ? latestInventory.namespaces : [])
|
||||
.map((group) => '<option value="' + escapeHtml(group.name) + '"></option>')
|
||||
.join('');
|
||||
detailsEl.innerHTML = [
|
||||
'<div class="stack">',
|
||||
'<div><h3 style="margin-bottom: 6px;">Namespace restore</h3><p class="muted" style="margin-top: 0;">Source namespace: ' + escapeHtml(namespace) + '</p></div>',
|
||||
'<label>Target namespace<input id="namespace-restore-target" list="namespace-restore-options" value="' + escapeHtml(namespace) + '"></label>',
|
||||
'<datalist id="namespace-restore-options">' + namespaceOptions + '</datalist>',
|
||||
'<label>Target PVC prefix<input id="namespace-restore-prefix" value="' + escapeHtml(suggestNamespaceRestorePrefix()) + '"></label>',
|
||||
'<label>Snapshot hint (optional, blank = latest completed per PVC)<input id="namespace-restore-snapshot" placeholder="blank uses latest"></label>',
|
||||
'<p class="muted">A target PVC will be created for each source PVC using this prefix.</p>',
|
||||
'<div class="actions">',
|
||||
'<button id="namespace-restore-run">Create restore PVCs</button>',
|
||||
'<button id="namespace-restore-dry" class="secondary">Dry run namespace restore</button>',
|
||||
'</div></div>'
|
||||
].join('');
|
||||
|
||||
const runRestore = async (dryRun) => {
|
||||
const targetNamespace = document.getElementById('namespace-restore-target').value.trim();
|
||||
const targetPrefix = document.getElementById('namespace-restore-prefix').value.trim();
|
||||
const snapshot = document.getElementById('namespace-restore-snapshot').value.trim();
|
||||
try {
|
||||
const payload = await fetchJSON('/v1/restores/namespace', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
namespace,
|
||||
target_namespace: targetNamespace,
|
||||
target_prefix: targetPrefix,
|
||||
snapshot,
|
||||
dry_run: dryRun
|
||||
})
|
||||
});
|
||||
showResult(payload);
|
||||
await loadInventory();
|
||||
} catch (error) {
|
||||
showResult({ error: error.message, namespace, target_namespace: targetNamespace, target_prefix: targetPrefix, dry_run: dryRun });
|
||||
}
|
||||
};
|
||||
|
||||
document.getElementById('namespace-restore-run').onclick = () => runRestore(false);
|
||||
document.getElementById('namespace-restore-dry').onclick = () => runRestore(true);
|
||||
}
|
||||
|
||||
async function showRestore(namespace, pvc) {
|
||||
detailsEl.innerHTML = '<p class="muted">Loading backups...</p>';
|
||||
try {
|
||||
@ -307,9 +414,91 @@
|
||||
}
|
||||
}
|
||||
|
||||
function renderPolicies(payload) {
|
||||
latestPolicies = payload && payload.policies ? payload.policies : [];
|
||||
if (!latestPolicies.length) {
|
||||
policyListEl.innerHTML = '<p class="muted">No policies yet. Add one to enable scheduled backups.</p>';
|
||||
return;
|
||||
}
|
||||
policyListEl.innerHTML = latestPolicies.map((policy) => {
|
||||
const scope = policy.pvc ? (policy.namespace + '/' + policy.pvc) : (policy.namespace + '/*');
|
||||
return [
|
||||
'<article class="policy-item stack">',
|
||||
'<div class="row" style="justify-content: space-between;">',
|
||||
'<span class="mono">' + escapeHtml(scope) + '</span>',
|
||||
'<span class="badge ' + (policy.enabled ? 'good' : 'bad') + '">' + (policy.enabled ? 'Enabled' : 'Disabled') + '</span>',
|
||||
'</div>',
|
||||
'<div class="meta">Every ' + escapeHtml(policy.interval_hours) + 'h</div>',
|
||||
'<div class="actions">',
|
||||
'<button class="secondary" data-action="policy-apply" data-namespace="' + escapeHtml(policy.namespace) + '" data-pvc="' + escapeHtml(policy.pvc || '') + '" data-interval="' + escapeHtml(policy.interval_hours) + '" data-enabled="' + escapeHtml(policy.enabled) + '">Load</button>',
|
||||
'<button class="secondary" data-action="policy-delete" data-policy-id="' + escapeHtml(policy.id) + '">Delete</button>',
|
||||
'</div>',
|
||||
'</article>'
|
||||
].join('');
|
||||
}).join('');
|
||||
|
||||
policyListEl.querySelectorAll('button[data-action="policy-delete"]').forEach((button) => {
|
||||
button.addEventListener('click', () => deletePolicy(button.dataset.policyId));
|
||||
});
|
||||
policyListEl.querySelectorAll('button[data-action="policy-apply"]').forEach((button) => {
|
||||
button.addEventListener('click', () => {
|
||||
policyNamespaceEl.value = button.dataset.namespace || '';
|
||||
policyPVCEl.value = button.dataset.pvc || '';
|
||||
policyIntervalEl.value = button.dataset.interval || '24';
|
||||
policyEnabledEl.checked = String(button.dataset.enabled) === 'true';
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function loadPolicies() {
|
||||
policyListEl.innerHTML = '<p class="muted">Refreshing policies...</p>';
|
||||
try {
|
||||
const payload = await fetchJSON('/v1/policies');
|
||||
renderPolicies(payload);
|
||||
} catch (error) {
|
||||
policyListEl.innerHTML = '<p class="error">' + escapeHtml(error.message) + '</p>';
|
||||
}
|
||||
}
|
||||
|
||||
async function savePolicy() {
|
||||
const namespace = policyNamespaceEl.value.trim();
|
||||
const pvc = policyPVCEl.value.trim();
|
||||
const intervalHours = Number(policyIntervalEl.value);
|
||||
const enabled = Boolean(policyEnabledEl.checked);
|
||||
try {
|
||||
const payload = await fetchJSON('/v1/policies', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
namespace,
|
||||
pvc,
|
||||
interval_hours: intervalHours,
|
||||
enabled
|
||||
})
|
||||
});
|
||||
showResult(payload);
|
||||
await loadPolicies();
|
||||
} catch (error) {
|
||||
showResult({ error: error.message, namespace, pvc, interval_hours: intervalHours, enabled });
|
||||
}
|
||||
}
|
||||
|
||||
async function deletePolicy(policyID) {
|
||||
try {
|
||||
const payload = await fetchJSON('/v1/policies/' + encodeURIComponent(policyID), { method: 'DELETE' });
|
||||
showResult(payload);
|
||||
await loadPolicies();
|
||||
} catch (error) {
|
||||
showResult({ error: error.message, policy_id: policyID });
|
||||
}
|
||||
}
|
||||
|
||||
function renderInventory(payload) {
|
||||
latestInventory = payload;
|
||||
generatedAtEl.textContent = payload.generated_at ? 'Updated ' + payload.generated_at : '';
|
||||
policyNamespaceOptionsEl.innerHTML = (payload.namespaces || [])
|
||||
.map((group) => '<option value="' + escapeHtml(group.name) + '"></option>')
|
||||
.join('');
|
||||
if (!payload.namespaces || payload.namespaces.length === 0) {
|
||||
inventoryEl.innerHTML = '<p class="muted">No bound PVCs found.</p>';
|
||||
return;
|
||||
@ -334,7 +523,18 @@
|
||||
'</div></article>'
|
||||
].join('');
|
||||
}).join('');
|
||||
return '<section class="namespace"><h3>' + escapeHtml(group.name) + '</h3>' + pvcs + '</section>';
|
||||
return [
|
||||
'<section class="namespace">',
|
||||
'<div class="row" style="justify-content: space-between; align-items: flex-start;">',
|
||||
'<h3 style="margin: 0;">' + escapeHtml(group.name) + '</h3>',
|
||||
'<div class="actions">',
|
||||
'<button class="secondary" data-action="backup-namespace" data-namespace="' + escapeHtml(group.name) + '">Backup namespace</button>',
|
||||
'<button class="secondary" data-action="restore-namespace" data-namespace="' + escapeHtml(group.name) + '">Restore namespace</button>',
|
||||
'</div>',
|
||||
'</div>',
|
||||
pvcs,
|
||||
'</section>'
|
||||
].join('');
|
||||
}).join('');
|
||||
|
||||
inventoryEl.querySelectorAll('button[data-action="backup"]').forEach((button) => {
|
||||
@ -343,6 +543,12 @@
|
||||
inventoryEl.querySelectorAll('button[data-action="restore"]').forEach((button) => {
|
||||
button.addEventListener('click', () => showRestore(button.dataset.namespace, button.dataset.pvc));
|
||||
});
|
||||
inventoryEl.querySelectorAll('button[data-action="backup-namespace"]').forEach((button) => {
|
||||
button.addEventListener('click', () => triggerNamespaceBackup(button.dataset.namespace, false));
|
||||
});
|
||||
inventoryEl.querySelectorAll('button[data-action="restore-namespace"]').forEach((button) => {
|
||||
button.addEventListener('click', () => showNamespaceRestore(button.dataset.namespace));
|
||||
});
|
||||
}
|
||||
|
||||
async function loadInventory() {
|
||||
@ -359,8 +565,10 @@
|
||||
}
|
||||
|
||||
refreshBtn.addEventListener('click', loadInventory);
|
||||
policySaveBtn.addEventListener('click', savePolicy);
|
||||
loadWhoAmI();
|
||||
loadInventory();
|
||||
loadPolicies();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user