package cluster import ( "context" "encoding/base64" "encoding/json" "fmt" "sort" "strings" "time" ) type hostPrivilegedAction string const ( hostActionSudoPreflight hostPrivilegedAction = "sudo-preflight" hostActionK3sAgentShow hostPrivilegedAction = "k3s-agent-show" hostActionK3sAgentIsActive hostPrivilegedAction = "k3s-agent-is-active" hostActionK3sAgentRestart hostPrivilegedAction = "k3s-agent-restart-no-block" hostActionHostReboot hostPrivilegedAction = "host-reboot" hostActionInstallCryptsetup hostPrivilegedAction = "install-cryptsetup-bin" hostActionModprobeDMCrypt hostPrivilegedAction = "modprobe-dm-crypt" hostActionCrictlPods hostPrivilegedAction = "crictl-pods" hostActionCrictlPs hostPrivilegedAction = "crictl-ps" hostActionCrictlInspect hostPrivilegedAction = "crictl-inspect" hostActionCrictlStop hostPrivilegedAction = "crictl-stop" ) type hostPrivilegeError struct { Class string Node string Action hostPrivilegedAction Detail string } // Error runs one orchestration or CLI step. // Signature: (e hostPrivilegeError) Error() string. // Why: host repair failures need stable operator-facing classes without // embedding secret material or raw command input. func (e hostPrivilegeError) Error() string { parts := []string{strings.TrimSpace(e.Class)} if strings.TrimSpace(e.Node) != "" { parts = append(parts, "node="+strings.TrimSpace(e.Node)) } if e.Action != "" { parts = append(parts, "action="+string(e.Action)) } if strings.TrimSpace(e.Detail) != "" { parts = append(parts, strings.TrimSpace(e.Detail)) } return strings.Join(parts, " ") } type hostSudoSecret struct { Data map[string]string `json:"data"` } // preflightHostPrivilege verifies that Ananke can run harmless sudo on a node. // Signature: (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error. // Why: host repair should fail with a clear privilege class before an outage // path needs a package install, k3s-agent restart, runtime inspection, or reboot. func (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error { _, err := o.runHostPrivilegedAction(ctx, node, hostActionSudoPreflight, 12*time.Second) return err } // runHostPrivilegedAction executes one allowlisted sudo action on a managed host. // Signature: (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error). // Why: recovery must not turn Vault-backed sudo into arbitrary remote root // execution; every privileged operation is classified, bounded, and auditable. func (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error) { node = strings.TrimSpace(node) if node == "" { return "", hostPrivilegeError{Class: "host-privilege-unavailable", Action: action, Detail: "node is empty"} } if !o.sshManaged(node) { return "", hostPrivilegeError{Class: "host-privilege-unavailable", Node: node, Action: action, Detail: "node is not SSH-managed"} } commandArgs, err := hostPrivilegedCommandArgs(action, args...) if err != nil { return "", err } if timeout <= 0 { timeout = o.hostPrivilegedCommandTimeout() } passwordless := sudoRemoteCommand(false, commandArgs) out, err := o.sshWithTimeout(ctx, node, passwordless, timeout) if err == nil { o.logHostPrivilegedAction(node, action, "passwordless-sudo") return out, nil } if !sudoMayNeedPassword(out, err) { return out, classifyHostPrivilegeFailure(node, action, err, out) } password, secretClass, secretErr := o.hostSudoPassword(ctx, node) if secretErr != nil { return "", hostPrivilegeError{ Class: "host-privilege-unavailable", Node: node, Action: action, Detail: fmt.Sprintf("%s: %v", secretClass, secretErr), } } passwordBacked := sudoRemoteCommand(true, commandArgs) out, err = o.sshWithInput(ctx, node, passwordBacked, password+"\n", timeout) if err != nil { return out, classifyHostPrivilegeFailure(node, action, err, out) } o.logHostPrivilegedAction(node, action, "vault-backed-sudo") return out, nil } // hostPrivilegedCommandTimeout runs one orchestration or CLI step. // Signature: (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration. // Why: every privileged host command must have a bounded default timeout even // when a focused unit test builds a partial config. func (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration { seconds := o.cfg.Startup.HostPrivilegedCommandTimeoutSec if seconds <= 0 { seconds = 90 } return time.Duration(seconds) * time.Second } // logHostPrivilegedAction runs one orchestration or CLI step. // Signature: (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string). // Why: audit logs should record the action class and sudo mode while omitting // command stdin, passwords, and secret contents. func (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string) { o.log.Printf("host privileged action completed node=%s action=%s mode=%s", node, action, mode) } // hostPrivilegedCommandArgs runs one orchestration or CLI step. // Signature: hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error). // Why: the Vault-backed sudo path must stay on a strict allowlist instead of // accepting arbitrary remote root commands. func hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error) { switch action { case hostActionSudoPreflight: return []string{"/usr/bin/systemctl", "--version"}, nil case hostActionK3sAgentShow: return []string{"systemctl", "show", "k3s-agent", "--property=ActiveState,SubState,Result,MainPID", "--no-pager"}, nil case hostActionK3sAgentIsActive: return []string{"systemctl", "is-active", "k3s-agent"}, nil case hostActionK3sAgentRestart: return []string{"systemctl", "--no-block", "restart", "k3s-agent"}, nil case hostActionHostReboot: return []string{"systemctl", "reboot"}, nil case hostActionInstallCryptsetup: return []string{ "env", "DEBIAN_FRONTEND=noninteractive", "sh", "-lc", "apt-get update && apt-get install -y --no-install-recommends cryptsetup-bin && (command -v cryptsetup >/dev/null 2>&1 || test -x /usr/sbin/cryptsetup || test -x /usr/bin/cryptsetup) && echo __ANANKE_CRYPTSETUP_INSTALLED__", }, nil case hostActionModprobeDMCrypt: return []string{"modprobe", "dm_crypt"}, nil case hostActionCrictlPods: return []string{"crictl", "pods", "-o", "json"}, nil case hostActionCrictlPs: return []string{"crictl", "ps", "-a", "-o", "json"}, nil case hostActionCrictlInspect: if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") { return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl inspect requires one safe container id"} } return []string{"crictl", "inspect", strings.TrimSpace(args[0])}, nil case hostActionCrictlStop: if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") { return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl stop requires one safe container id"} } return []string{"crictl", "stop", strings.TrimSpace(args[0])}, nil default: return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "action is not allowlisted"} } } // sudoRemoteCommand runs one orchestration or CLI step. // Signature: sudoRemoteCommand(passwordBacked bool, commandArgs []string) string. // Why: passwordless and password-backed sudo share one command builder so // quoting and prompt suppression stay consistent. func sudoRemoteCommand(passwordBacked bool, commandArgs []string) string { parts := []string{"sudo"} if passwordBacked { parts = append(parts, "-S", "-p", "") } else { parts = append(parts, "-n") } parts = append(parts, commandArgs...) return shellJoin(parts) } // shellJoin runs one orchestration or CLI step. // Signature: shellJoin(parts []string) string. // Why: remote SSH commands need readable simple words and safe quoting for // shell-sensitive fragments such as package-manager scripts. func shellJoin(parts []string) string { out := make([]string, 0, len(parts)) for _, part := range parts { out = append(out, shellQuoteIfNeeded(part)) } return strings.Join(out, " ") } // shellQuoteIfNeeded runs one orchestration or CLI step. // Signature: shellQuoteIfNeeded(part string) string. // Why: audit-friendly commands should avoid unnecessary quotes while still // protecting whitespace, quotes, and metacharacters. func shellQuoteIfNeeded(part string) string { if part == "" { return shellQuote(part) } for _, r := range part { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { continue } switch r { case '_', '-', '.', '/', '=', ':': continue default: return shellQuote(part) } } return part } // sudoMayNeedPassword runs one orchestration or CLI step. // Signature: sudoMayNeedPassword(out string, err error) bool. // Why: Ananke should try Vault-backed sudo only for sudo privilege failures, not // for unrelated SSH or command errors. func sudoMayNeedPassword(out string, err error) bool { full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err))) needles := []string{ "a password is required", "password is required", "sudo: a terminal is required", "no tty present", "may not run sudo", "sudo denied", "is not in the sudoers file", } for _, needle := range needles { if strings.Contains(full, needle) { return true } } return false } // classifyHostPrivilegeFailure runs one orchestration or CLI step. // Signature: classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error. // Why: host repair needs distinct timeout, auth, SSH, and command-failure // classes so status output tells the operator what kind of help is needed. func classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error { full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err))) class := "host-command-failed" switch { case strings.Contains(full, "context deadline exceeded"), strings.Contains(full, "timed out"), strings.Contains(full, "timeout"): class = "host-command-timeout" case strings.Contains(full, "authentication failure"), strings.Contains(full, "incorrect password"), strings.Contains(full, "try again"): class = "host-privilege-auth-failed" case strings.Contains(full, "permission denied"), strings.Contains(full, "publickey"): class = "host-ssh-unavailable" } return hostPrivilegeError{Class: class, Node: node, Action: action, Detail: scrubHostPrivilegeDetail(out, err)} } // scrubHostPrivilegeDetail runs one orchestration or CLI step. // Signature: scrubHostPrivilegeDetail(out string, err error) string. // Why: host command details should be compact enough for annotations and avoid // accidental multi-line or control-character log noise. func scrubHostPrivilegeDetail(out string, err error) string { detail := strings.TrimSpace(fmt.Sprint(err)) if trimmed := strings.TrimSpace(out); trimmed != "" { if detail != "" { detail += ": " } detail += trimmed } return sanitizeCordonAnnotationValue(detail) } // hostSudoPassword runs one orchestration or CLI step. // Signature: (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error). // Why: sudo material should come from a configured Vault-synced Kubernetes // Secret and never be logged or passed through ordinary command arguments. func (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error) { namespace := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNamespace) template := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNameTemplate) key := strings.TrimSpace(o.cfg.Startup.HostSudoSecretPasswordKey) if key == "" { key = "password" } if namespace == "" || template == "" { return "", "secret-lookup-unconfigured", fmt.Errorf("host sudo secret namespace/template is not configured") } secretName := strings.ReplaceAll(template, "{node}", node) secretName = strings.TrimSpace(secretName) if secretName == "" { return "", "secret-lookup-invalid", fmt.Errorf("host sudo secret name resolved empty") } out, err := o.kubectl(ctx, 15*time.Second, "-n", namespace, "get", "secret", secretName, "-o", "json") if err != nil { return "", "secret-lookup-failed", err } var secret hostSudoSecret if err := json.Unmarshal([]byte(out), &secret); err != nil { return "", "secret-decode-failed", err } encoded := strings.TrimSpace(secret.Data[key]) if encoded == "" { keys := make([]string, 0, len(secret.Data)) for k := range secret.Data { keys = append(keys, k) } sort.Strings(keys) return "", "secret-key-missing", fmt.Errorf("password key %q missing (available=%s)", key, joinLimited(keys, 4)) } decoded, err := base64.StdEncoding.DecodeString(encoded) if err != nil { return "", "secret-decode-failed", err } password := strings.TrimRight(string(decoded), "\r\n") if password == "" { return "", "secret-empty", fmt.Errorf("password key %q is empty", key) } return password, "secret-lookup-ok", nil }