ananke/internal/cluster/orchestrator_critical_probe_repair.go

package cluster

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"
)

type criticalBackendWorkload struct {
	Spec struct {
		Template struct {
			Spec struct {
				Containers []criticalBackendContainer `json:"containers"`
			} `json:"spec"`
		} `json:"template"`
	} `json:"spec"`
}

type criticalBackendContainer struct {
	Name          string         `json:"name"`
	LivenessProbe map[string]any `json:"livenessProbe"`
	StartupProbe  map[string]any `json:"startupProbe"`
}

// maybeRepairCriticalBackendStartupProbes runs one orchestration or CLI step.
// Signature: (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error).
// Why: critical backends that are slow after outage recovery can be killed by
// liveness probes before they ever become ready; adding a startup probe lets
// Kubernetes distinguish slow boot from dead process without app-specific code.
func (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error) {
	if !o.cfg.Startup.CriticalServiceStartupProbeRepair {
		return nil, nil
	}
	namespace = strings.TrimSpace(namespace)
	service = strings.TrimSpace(service)
	if namespace == "" || service == "" {
		return nil, nil
	}
	repaired := []string{}
	errs := []string{}
	for _, kind := range []string{"deployment", "statefulset"} {
		items, err := o.repairWorkloadStartupProbes(ctx, namespace, kind, service)
		if err != nil {
			if isNotFoundErr(err) {
				continue
			}
			errs = append(errs, err.Error())
			continue
		}
		repaired = append(repaired, items...)
	}
	if len(errs) > 0 {
		return repaired, errors.New(strings.Join(errs, "; "))
	}
	return repaired, nil
}

// repairWorkloadStartupProbes runs one orchestration or CLI step.
// Signature: (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error).
// Why: the startup-probe patch is deliberately limited to an existing
// deployment/statefulset with a liveness probe and no startup probe.
func (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error) {
	out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", kind, name, "-o", "json")
	if err != nil {
		if strings.TrimSpace(out) != "" {
			return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(out))
		}
		return nil, err
	}
	var workload criticalBackendWorkload
	if err := json.Unmarshal([]byte(out), &workload); err != nil {
		return nil, fmt.Errorf("decode %s/%s/%s for startup probe repair: %w", namespace, kind, name, err)
	}

	containers := []map[string]any{}
	threshold := o.cfg.Startup.CriticalServiceStartupProbeThreshold
	if threshold <= 0 {
		threshold = 30
	}
	for _, container := range workload.Spec.Template.Spec.Containers {
		containerName := strings.TrimSpace(container.Name)
		if containerName == "" || len(container.LivenessProbe) == 0 || len(container.StartupProbe) > 0 {
			continue
		}
		startupProbe := cloneMap(container.LivenessProbe)
		startupProbe["failureThreshold"] = threshold
		if _, ok := startupProbe["successThreshold"]; !ok {
			startupProbe["successThreshold"] = 1
		}
		containers = append(containers, map[string]any{
			"name":         containerName,
			"startupProbe": startupProbe,
		})
	}
	if len(containers) == 0 {
		return nil, nil
	}

	now := time.Now().UTC().Format(time.RFC3339)
	patch := map[string]any{
		"spec": map[string]any{
			"template": map[string]any{
				"metadata": map[string]any{
					"annotations": map[string]string{
						"ananke.bstein.dev/startup-probe-repair-at":     now,
						"ananke.bstein.dev/startup-probe-repair-reason": "critical-service-backend-unready",
					},
				},
				"spec": map[string]any{
					"containers": containers,
				},
			},
		},
	}
	rawPatch, err := json.Marshal(patch)
	if err != nil {
		return nil, fmt.Errorf("encode startup probe repair patch: %w", err)
	}
	if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "patch", kind, name, "--type=strategic", "-p", string(rawPatch)); err != nil {
		return nil, fmt.Errorf("patch startup probes on %s/%s/%s: %w", namespace, kind, name, err)
	}
	o.log.Printf("repaired startup probes on critical backend %s/%s/%s containers=%d", namespace, kind, name, len(containers))
	o.noteStartupAutoHeal(fmt.Sprintf("repaired startup probes on %s/%s/%s", namespace, kind, name))
	return []string{namespace + "/" + kind + "/" + name}, nil
}

// cloneMap runs one orchestration or CLI step.
// Signature: cloneMap(in map[string]any) map[string]any.
// Why: Kubernetes probe objects are patch payloads; cloning avoids mutating the
// decoded workload fixture while changing the generated startup probe threshold.
func cloneMap(in map[string]any) map[string]any {
	out := map[string]any{}
	for key, value := range in {
		if nested, ok := value.(map[string]any); ok {
			out[key] = cloneMap(nested)
			continue
		}
		out[key] = value
	}
	return out
}