ananke/internal/cluster/orchestrator_host_privilege.go

package cluster

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"
)

type hostPrivilegedAction string

const (
	hostActionSudoPreflight     hostPrivilegedAction = "sudo-preflight"
	hostActionK3sAgentShow      hostPrivilegedAction = "k3s-agent-show"
	hostActionK3sAgentIsActive  hostPrivilegedAction = "k3s-agent-is-active"
	hostActionK3sAgentRestart   hostPrivilegedAction = "k3s-agent-restart-no-block"
	hostActionHostReboot        hostPrivilegedAction = "host-reboot"
	hostActionInstallCryptsetup hostPrivilegedAction = "install-cryptsetup-bin"
	hostActionModprobeDMCrypt   hostPrivilegedAction = "modprobe-dm-crypt"
	hostActionCrictlPods        hostPrivilegedAction = "crictl-pods"
	hostActionCrictlPs          hostPrivilegedAction = "crictl-ps"
	hostActionCrictlInspect     hostPrivilegedAction = "crictl-inspect"
	hostActionCrictlStop        hostPrivilegedAction = "crictl-stop"
)

type hostPrivilegeError struct {
	Class  string
	Node   string
	Action hostPrivilegedAction
	Detail string
}

// Error runs one orchestration or CLI step.
// Signature: (e hostPrivilegeError) Error() string.
// Why: host repair failures need stable operator-facing classes without
// embedding secret material or raw command input.
func (e hostPrivilegeError) Error() string {
	parts := []string{strings.TrimSpace(e.Class)}
	if strings.TrimSpace(e.Node) != "" {
		parts = append(parts, "node="+strings.TrimSpace(e.Node))
	}
	if e.Action != "" {
		parts = append(parts, "action="+string(e.Action))
	}
	if strings.TrimSpace(e.Detail) != "" {
		parts = append(parts, strings.TrimSpace(e.Detail))
	}
	return strings.Join(parts, " ")
}

type hostSudoSecret struct {
	Data map[string]string `json:"data"`
}

// preflightHostPrivilege verifies that Ananke can run harmless sudo on a node.
// Signature: (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error.
// Why: host repair should fail with a clear privilege class before an outage
// path needs a package install, k3s-agent restart, runtime inspection, or reboot.
func (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error {
	_, err := o.runHostPrivilegedAction(ctx, node, hostActionSudoPreflight, 12*time.Second)
	return err
}

// runHostPrivilegedAction executes one allowlisted sudo action on a managed host.
// Signature: (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error).
// Why: recovery must not turn Vault-backed sudo into arbitrary remote root
// execution; every privileged operation is classified, bounded, and auditable.
func (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error) {
	node = strings.TrimSpace(node)
	if node == "" {
		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Action: action, Detail: "node is empty"}
	}
	if !o.sshManaged(node) {
		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Node: node, Action: action, Detail: "node is not SSH-managed"}
	}
	commandArgs, err := hostPrivilegedCommandArgs(action, args...)
	if err != nil {
		return "", err
	}
	if timeout <= 0 {
		timeout = o.hostPrivilegedCommandTimeout()
	}

	passwordless := sudoRemoteCommand(false, commandArgs)
	out, err := o.sshWithTimeout(ctx, node, passwordless, timeout)
	if err == nil {
		o.logHostPrivilegedAction(node, action, "passwordless-sudo")
		return out, nil
	}
	if !sudoMayNeedPassword(out, err) {
		return out, classifyHostPrivilegeFailure(node, action, err, out)
	}

	password, secretClass, secretErr := o.hostSudoPassword(ctx, node)
	if secretErr != nil {
		return "", hostPrivilegeError{
			Class:  "host-privilege-unavailable",
			Node:   node,
			Action: action,
			Detail: fmt.Sprintf("%s: %v", secretClass, secretErr),
		}
	}
	passwordBacked := sudoRemoteCommand(true, commandArgs)
	out, err = o.sshWithInput(ctx, node, passwordBacked, password+"\n", timeout)
	if err != nil {
		return out, classifyHostPrivilegeFailure(node, action, err, out)
	}
	o.logHostPrivilegedAction(node, action, "vault-backed-sudo")
	return out, nil
}

// hostPrivilegedCommandTimeout runs one orchestration or CLI step.
// Signature: (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration.
// Why: every privileged host command must have a bounded default timeout even
// when a focused unit test builds a partial config.
func (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration {
	seconds := o.cfg.Startup.HostPrivilegedCommandTimeoutSec
	if seconds <= 0 {
		seconds = 90
	}
	return time.Duration(seconds) * time.Second
}

// logHostPrivilegedAction runs one orchestration or CLI step.
// Signature: (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string).
// Why: audit logs should record the action class and sudo mode while omitting
// command stdin, passwords, and secret contents.
func (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string) {
	o.log.Printf("host privileged action completed node=%s action=%s mode=%s", node, action, mode)
}

// hostPrivilegedCommandArgs runs one orchestration or CLI step.
// Signature: hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error).
// Why: the Vault-backed sudo path must stay on a strict allowlist instead of
// accepting arbitrary remote root commands.
func hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error) {
	switch action {
	case hostActionSudoPreflight:
		return []string{"/usr/bin/systemctl", "--version"}, nil
	case hostActionK3sAgentShow:
		return []string{"systemctl", "show", "k3s-agent", "--property=ActiveState,SubState,Result,MainPID", "--no-pager"}, nil
	case hostActionK3sAgentIsActive:
		return []string{"systemctl", "is-active", "k3s-agent"}, nil
	case hostActionK3sAgentRestart:
		return []string{"systemctl", "--no-block", "restart", "k3s-agent"}, nil
	case hostActionHostReboot:
		return []string{"systemctl", "reboot"}, nil
	case hostActionInstallCryptsetup:
		return []string{
			"env",
			"DEBIAN_FRONTEND=noninteractive",
			"sh",
			"-lc",
			"apt-get update && apt-get install -y --no-install-recommends cryptsetup-bin && (command -v cryptsetup >/dev/null 2>&1 || test -x /usr/sbin/cryptsetup || test -x /usr/bin/cryptsetup) && echo __ANANKE_CRYPTSETUP_INSTALLED__",
		}, nil
	case hostActionModprobeDMCrypt:
		return []string{"modprobe", "dm_crypt"}, nil
	case hostActionCrictlPods:
		return []string{"crictl", "pods", "-o", "json"}, nil
	case hostActionCrictlPs:
		return []string{"crictl", "ps", "-a", "-o", "json"}, nil
	case hostActionCrictlInspect:
		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl inspect requires one safe container id"}
		}
		return []string{"crictl", "inspect", strings.TrimSpace(args[0])}, nil
	case hostActionCrictlStop:
		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl stop requires one safe container id"}
		}
		return []string{"crictl", "stop", strings.TrimSpace(args[0])}, nil
	default:
		return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "action is not allowlisted"}
	}
}

// sudoRemoteCommand runs one orchestration or CLI step.
// Signature: sudoRemoteCommand(passwordBacked bool, commandArgs []string) string.
// Why: passwordless and password-backed sudo share one command builder so
// quoting and prompt suppression stay consistent.
func sudoRemoteCommand(passwordBacked bool, commandArgs []string) string {
	parts := []string{"sudo"}
	if passwordBacked {
		parts = append(parts, "-S", "-p", "")
	} else {
		parts = append(parts, "-n")
	}
	parts = append(parts, commandArgs...)
	return shellJoin(parts)
}

// shellJoin runs one orchestration or CLI step.
// Signature: shellJoin(parts []string) string.
// Why: remote SSH commands need readable simple words and safe quoting for
// shell-sensitive fragments such as package-manager scripts.
func shellJoin(parts []string) string {
	out := make([]string, 0, len(parts))
	for _, part := range parts {
		out = append(out, shellQuoteIfNeeded(part))
	}
	return strings.Join(out, " ")
}

// shellQuoteIfNeeded runs one orchestration or CLI step.
// Signature: shellQuoteIfNeeded(part string) string.
// Why: audit-friendly commands should avoid unnecessary quotes while still
// protecting whitespace, quotes, and metacharacters.
func shellQuoteIfNeeded(part string) string {
	if part == "" {
		return shellQuote(part)
	}
	for _, r := range part {
		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
			continue
		}
		switch r {
		case '_', '-', '.', '/', '=', ':':
			continue
		default:
			return shellQuote(part)
		}
	}
	return part
}

// sudoMayNeedPassword runs one orchestration or CLI step.
// Signature: sudoMayNeedPassword(out string, err error) bool.
// Why: Ananke should try Vault-backed sudo only for sudo privilege failures, not
// for unrelated SSH or command errors.
func sudoMayNeedPassword(out string, err error) bool {
	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
	needles := []string{
		"a password is required",
		"password is required",
		"sudo: a terminal is required",
		"no tty present",
		"may not run sudo",
		"sudo denied",
		"is not in the sudoers file",
	}
	for _, needle := range needles {
		if strings.Contains(full, needle) {
			return true
		}
	}
	return false
}

// classifyHostPrivilegeFailure runs one orchestration or CLI step.
// Signature: classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error.
// Why: host repair needs distinct timeout, auth, SSH, and command-failure
// classes so status output tells the operator what kind of help is needed.
func classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error {
	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
	class := "host-command-failed"
	switch {
	case strings.Contains(full, "context deadline exceeded"), strings.Contains(full, "timed out"), strings.Contains(full, "timeout"):
		class = "host-command-timeout"
	case strings.Contains(full, "authentication failure"), strings.Contains(full, "incorrect password"), strings.Contains(full, "try again"):
		class = "host-privilege-auth-failed"
	case strings.Contains(full, "permission denied"), strings.Contains(full, "publickey"):
		class = "host-ssh-unavailable"
	}
	return hostPrivilegeError{Class: class, Node: node, Action: action, Detail: scrubHostPrivilegeDetail(out, err)}
}

// scrubHostPrivilegeDetail runs one orchestration or CLI step.
// Signature: scrubHostPrivilegeDetail(out string, err error) string.
// Why: host command details should be compact enough for annotations and avoid
// accidental multi-line or control-character log noise.
func scrubHostPrivilegeDetail(out string, err error) string {
	detail := strings.TrimSpace(fmt.Sprint(err))
	if trimmed := strings.TrimSpace(out); trimmed != "" {
		if detail != "" {
			detail += ": "
		}
		detail += trimmed
	}
	return sanitizeCordonAnnotationValue(detail)
}

// hostSudoPassword runs one orchestration or CLI step.
// Signature: (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error).
// Why: sudo material should come from a configured Vault-synced Kubernetes
// Secret and never be logged or passed through ordinary command arguments.
func (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error) {
	namespace := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNamespace)
	template := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNameTemplate)
	key := strings.TrimSpace(o.cfg.Startup.HostSudoSecretPasswordKey)
	if key == "" {
		key = "password"
	}
	if namespace == "" || template == "" {
		return "", "secret-lookup-unconfigured", fmt.Errorf("host sudo secret namespace/template is not configured")
	}
	secretName := strings.ReplaceAll(template, "{node}", node)
	secretName = strings.TrimSpace(secretName)
	if secretName == "" {
		return "", "secret-lookup-invalid", fmt.Errorf("host sudo secret name resolved empty")
	}

	out, err := o.kubectl(ctx, 15*time.Second, "-n", namespace, "get", "secret", secretName, "-o", "json")
	if err != nil {
		return "", "secret-lookup-failed", err
	}
	var secret hostSudoSecret
	if err := json.Unmarshal([]byte(out), &secret); err != nil {
		return "", "secret-decode-failed", err
	}
	encoded := strings.TrimSpace(secret.Data[key])
	if encoded == "" {
		keys := make([]string, 0, len(secret.Data))
		for k := range secret.Data {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		return "", "secret-key-missing", fmt.Errorf("password key %q missing (available=%s)", key, joinLimited(keys, 4))
	}
	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return "", "secret-decode-failed", err
	}
	password := strings.TrimRight(string(decoded), "\r\n")
	if password == "" {
		return "", "secret-empty", fmt.Errorf("password key %q is empty", key)
	}
	return password, "secret-lookup-ok", nil
}