ananke/cmd/ananke/bootstrap_handoff.go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/config"
	"scm.bstein.dev/bstein/ananke/internal/sshutil"
	"scm.bstein.dev/bstein/ananke/internal/state"
)

var (
	sshConfigCandidates = []string{
		"/home/atlas/.ssh/config",
		"/home/tethys/.ssh/config",
	}
	sshIdentityCandidates = []string{
		"/home/atlas/.ssh/id_ed25519",
		"/home/tethys/.ssh/id_ed25519",
	}
)

// tryPeerBootstrapHandoff runs one orchestration or CLI step.
// Signature: tryPeerBootstrapHandoff(ctx context.Context, cfg config.Config, logger *log.Logger) (bool, error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func tryPeerBootstrapHandoff(ctx context.Context, cfg config.Config, logger *log.Logger) (bool, error) {
	coordinator := strings.TrimSpace(cfg.Coordination.ForwardShutdownHost)
	if coordinator == "" {
		return false, fmt.Errorf("coordination.forward_shutdown_host is empty for peer role")
	}
	user := strings.TrimSpace(cfg.Coordination.ForwardShutdownUser)
	if user == "" {
		if override, ok := cfg.SSHNodeUsers[coordinator]; ok && strings.TrimSpace(override) != "" {
			user = strings.TrimSpace(override)
		} else {
			user = strings.TrimSpace(cfg.SSHUser)
		}
	}

	host := coordinator
	if mapped, ok := cfg.SSHNodeHosts[coordinator]; ok && strings.TrimSpace(mapped) != "" {
		host = strings.TrimSpace(mapped)
	}
	target := host
	if user != "" {
		target = user + "@" + host
	}

	args := buildSSHBaseArgs(cfg)

	remote := "sudo -n systemctl start ananke-bootstrap.service"
	attempt := 1
	for {
		cmdArgs := append(append([]string{}, args...), target, remote)
		_, err := runSSHWithRecovery(ctx, logger, cfg, cmdArgs, []string{coordinator, host, cfg.SSHJumpHost})
		if err == nil {
			logger.Printf("peer bootstrap handoff succeeded on %s (attempt=%d)", coordinator, attempt)
			return true, nil
		}
		logger.Printf("peer bootstrap handoff attempt %d failed for %s: %v", attempt, coordinator, err)

		select {
		case <-ctx.Done():
			return false, fmt.Errorf("coordinator handoff timeout for %s: %w", coordinator, ctx.Err())
		case <-time.After(5 * time.Second):
			attempt++
		}
	}
}

// coordinatorAllowsPeerFallbackStartup runs one orchestration or CLI step.
// Signature: coordinatorAllowsPeerFallbackStartup(ctx context.Context, cfg config.Config, logger *log.Logger) (bool, string, error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func coordinatorAllowsPeerFallbackStartup(ctx context.Context, cfg config.Config, logger *log.Logger) (bool, string, error) {
	coordinator := strings.TrimSpace(cfg.Coordination.ForwardShutdownHost)
	if coordinator == "" {
		return true, "no coordinator configured", nil
	}
	user := strings.TrimSpace(cfg.Coordination.ForwardShutdownUser)
	if user == "" {
		if override, ok := cfg.SSHNodeUsers[coordinator]; ok && strings.TrimSpace(override) != "" {
			user = strings.TrimSpace(override)
		} else {
			user = strings.TrimSpace(cfg.SSHUser)
		}
	}
	host := coordinator
	if mapped, ok := cfg.SSHNodeHosts[coordinator]; ok && strings.TrimSpace(mapped) != "" {
		host = strings.TrimSpace(mapped)
	}
	target := host
	if user != "" {
		target = user + "@" + host
	}
	remoteCmd := "if sudo -n /usr/bin/systemctl is-active --quiet ananke-bootstrap.service; then echo __ANANKE_BOOTSTRAP_ACTIVE__; else echo __ANANKE_BOOTSTRAP_IDLE__; fi; sudo -n /usr/local/bin/ananke intent --config /etc/ananke/ananke.yaml"
	args := append(buildSSHBaseArgs(cfg), target, remoteCmd)
	out, err := runSSHWithRecovery(ctx, logger, cfg, args, []string{coordinator, host, cfg.SSHJumpHost})
	if err != nil {
		logger.Printf("warning: coordinator guard check unavailable on %s: %v; allowing peer fallback startup", coordinator, err)
		return true, "coordinator unreachable", nil
	}
	trimmed := strings.TrimSpace(out)
	if strings.Contains(trimmed, "__ANANKE_BOOTSTRAP_ACTIVE__") {
		return false, "coordinator bootstrap service is active", nil
	}
	remoteIntent, parseErr := state.ParseIntentOutput(trimmed)
	if parseErr != nil {
		return false, "", fmt.Errorf("decode coordinator intent: %w", parseErr)
	}
	if remoteIntent.State == "" || remoteIntent.State == state.IntentNormal {
		return true, "coordinator intent is normal", nil
	}
	guardAge := time.Duration(maxInt(cfg.Coordination.StartupGuardMaxAgeSec, 60)) * time.Second
	intentAge := time.Duration(0)
	if !remoteIntent.UpdatedAt.IsZero() {
		intentAge = time.Since(remoteIntent.UpdatedAt)
	}
	switch remoteIntent.State {
	case state.IntentShuttingDown:
		if remoteIntent.UpdatedAt.IsZero() || intentAge <= guardAge {
			return false, fmt.Sprintf("coordinator intent=%s age=%s reason=%q", remoteIntent.State, intentAge.Round(time.Second), remoteIntent.Reason), nil
		}
		logger.Printf("warning: coordinator shutdown intent appears stale (age=%s > guard=%s); allowing peer fallback startup", intentAge.Round(time.Second), guardAge)
		return true, "coordinator shutdown intent stale", nil
	case state.IntentStartupInProgress:
		if remoteIntent.UpdatedAt.IsZero() || intentAge <= guardAge {
			return false, fmt.Sprintf("coordinator intent=%s age=%s reason=%q", remoteIntent.State, intentAge.Round(time.Second), remoteIntent.Reason), nil
		}
		logger.Printf("warning: coordinator startup intent appears stale (age=%s > guard=%s); allowing peer fallback startup", intentAge.Round(time.Second), guardAge)
		return true, "coordinator startup intent stale", nil
	case state.IntentShutdownComplete:
		if remoteIntent.UpdatedAt.IsZero() {
			return false, "coordinator reported shutdown_complete with unknown age", nil
		}
		if intentAge <= startupShutdownCooldown(cfg) {
			return false, fmt.Sprintf("coordinator recently completed shutdown (%s ago)", intentAge.Round(time.Second)), nil
		}
		return true, "coordinator shutdown_complete is old enough", nil
	default:
		return false, fmt.Sprintf("coordinator intent state %q is unknown", remoteIntent.State), nil
	}
}

// runSSHWithRecovery runs one orchestration or CLI step.
// Signature: runSSHWithRecovery(ctx context.Context, logger *log.Logger, cfg config.Config, args []string, repairHosts []string) (string, error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func runSSHWithRecovery(ctx context.Context, logger *log.Logger, cfg config.Config, args []string, repairHosts []string) (string, error) {
	try := func() (string, error) {
		cmd := exec.CommandContext(ctx, "ssh", args...)
		out, err := cmd.CombinedOutput()
		trimmed := strings.TrimSpace(string(out))
		if err != nil {
			if trimmed == "" {
				return "", fmt.Errorf("ssh failed: %w", err)
			}
			return trimmed, fmt.Errorf("ssh failed: %w: %s", err, trimmed)
		}
		return trimmed, nil
	}

	out, err := try()
	if err == nil {
		return out, nil
	}
	if !sshutil.ShouldAttemptKnownHostsRepair(out, err) {
		return out, err
	}

	sshutil.RepairKnownHosts(ctx, logger, sshutil.KnownHostsFiles(resolveSSHConfigFile(cfg), resolveSSHIdentityFile(cfg)), repairHosts, cfg.SSHPort)
	return try()
}

// buildSSHBaseArgs runs one orchestration or CLI step.
// Signature: buildSSHBaseArgs(cfg config.Config) []string.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func buildSSHBaseArgs(cfg config.Config) []string {
	args := []string{
		"-o", "BatchMode=yes",
		"-o", "ConnectTimeout=8",
		"-o", "StrictHostKeyChecking=accept-new",
	}
	if cfgPath := resolveSSHConfigFile(cfg); cfgPath != "" {
		args = append(args, "-F", cfgPath)
	}
	if idPath := resolveSSHIdentityFile(cfg); idPath != "" {
		args = append(args, "-i", idPath)
	}
	if cfg.SSHPort > 0 {
		args = append(args, "-p", strconv.Itoa(cfg.SSHPort))
	}
	if cfg.SSHJumpHost != "" {
		jump := cfg.SSHJumpHost
		if cfg.SSHJumpUser != "" {
			jump = cfg.SSHJumpUser + "@" + jump
		}
		if cfg.SSHPort > 0 && !strings.Contains(jump, ":") {
			jump = fmt.Sprintf("%s:%d", jump, cfg.SSHPort)
		}
		args = append(args, "-J", jump)
	}
	return args
}

// resolveSSHConfigFile runs one orchestration or CLI step.
// Signature: resolveSSHConfigFile(cfg config.Config) string.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func resolveSSHConfigFile(cfg config.Config) string {
	if strings.TrimSpace(cfg.SSHConfigFile) != "" {
		return strings.TrimSpace(cfg.SSHConfigFile)
	}
	for _, p := range sshConfigCandidates {
		if stat, err := os.Stat(p); err == nil && !stat.IsDir() {
			return p
		}
	}
	return ""
}

// resolveSSHIdentityFile runs one orchestration or CLI step.
// Signature: resolveSSHIdentityFile(cfg config.Config) string.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func resolveSSHIdentityFile(cfg config.Config) string {
	if strings.TrimSpace(cfg.SSHIdentityFile) != "" {
		return strings.TrimSpace(cfg.SSHIdentityFile)
	}
	for _, p := range sshIdentityCandidates {
		if stat, err := os.Stat(p); err == nil && !stat.IsDir() {
			return p
		}
	}
	return ""
}

// maxInt runs one orchestration or CLI step.
// Signature: maxInt(a, b int) int.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func maxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// startupShutdownCooldown runs one orchestration or CLI step.
// Signature: startupShutdownCooldown(cfg config.Config) time.Duration.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func startupShutdownCooldown(cfg config.Config) time.Duration {
	seconds := cfg.Startup.ShutdownCooldownSeconds
	if seconds <= 0 {
		seconds = 45
	}
	return time.Duration(seconds) * time.Second
}