ananke/internal/cluster/orchestrator_tcp_service_check.go

package cluster

import (
	"bufio"
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"sort"
	"strings"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/config"
)

// tcpServiceChecklistReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string).
// Why: SMTP, IMAP, and other non-HTTP services need protocol checks alongside
// HTTP ingress checks before startup can call the cluster healthy.
func (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string) {
	checks := o.cfg.Startup.TCPServiceChecklist
	if len(checks) == 0 {
		return true, "no tcp checklist items configured"
	}
	for _, check := range checks {
		ok, detail := o.tcpServiceCheckReady(ctx, check)
		if !ok {
			name := strings.TrimSpace(check.Name)
			if name == "" {
				name = fmt.Sprintf("%s:%d", strings.TrimSpace(check.Host), check.Port)
			}
			return false, fmt.Sprintf("%s: %s", name, detail)
		}
	}
	return true, fmt.Sprintf("tcp-checks=%d", len(checks))
}

// tcpServiceCheckReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string).
// Why: a single mail protocol endpoint should report whether connect, TLS, and
// banner expectations passed without exposing any configured secret material.
func (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string) {
	banner, err := tcpServiceProbe(ctx, check)
	if err != nil {
		return false, err.Error()
	}
	expected := strings.TrimSpace(check.ExpectContains)
	if expected != "" && !checklistContains(banner, expected) {
		return false, fmt.Sprintf("banner missing expected marker %q", expected)
	}
	return true, "connected"
}

// tcpServiceProbe runs one orchestration or CLI step.
// Signature: tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error).
// Why: protocol probes need a small direct TCP/TLS implementation so Ananke can
// validate Mailu without shelling out to nc or openssl.
func tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error) {
	timeout := time.Duration(check.TimeoutSeconds) * time.Second
	if timeout <= 0 {
		timeout = 8 * time.Second
	}
	address := net.JoinHostPort(strings.TrimSpace(check.Host), fmt.Sprintf("%d", check.Port))
	dialer := net.Dialer{Timeout: timeout}
	conn, err := dialer.DialContext(ctx, "tcp", address)
	if err != nil {
		return "", fmt.Errorf("connect %s failed: %w", address, err)
	}
	defer conn.Close()
	deadline := time.Now().Add(timeout)
	_ = conn.SetDeadline(deadline)

	if check.TLS {
		tlsConn := tls.Client(conn, &tls.Config{
			ServerName:         strings.TrimSpace(check.Host),
			InsecureSkipVerify: check.InsecureSkipTLS,
		})
		if err := tlsConn.HandshakeContext(ctx); err != nil {
			return "", fmt.Errorf("tls handshake %s failed: %w", address, err)
		}
		conn = tlsConn
		_ = conn.SetDeadline(deadline)
	}

	reader := bufio.NewReader(conn)
	bannerParts := []string{}
	line, _ := reader.ReadString('\n')
	if strings.TrimSpace(line) != "" {
		bannerParts = append(bannerParts, strings.TrimSpace(line))
	}
	if send := check.Send; send != "" {
		if _, err := conn.Write([]byte(send)); err != nil {
			return strings.Join(bannerParts, "\n"), fmt.Errorf("write probe command failed: %w", err)
		}
		line, _ = reader.ReadString('\n')
		if strings.TrimSpace(line) != "" {
			bannerParts = append(bannerParts, strings.TrimSpace(line))
		}
	}
	return strings.Join(bannerParts, "\n"), nil
}

// maybeAutoHealTCPServiceBackends runs one orchestration or CLI step.
// Signature: (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time).
// Why: failed Mailu protocol probes should drive the same backend repair loop as
// HTTP services when a Kubernetes service hint is configured.
func (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time) {
	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
		return
	}
	now := time.Now()
	if lastAttempt != nil && !lastAttempt.IsZero() && now.Sub(*lastAttempt) < 45*time.Second {
		return
	}
	if lastAttempt != nil {
		*lastAttempt = now
	}
	healed, err := o.healFailedTCPServiceBackends(ctx)
	if err != nil {
		o.log.Printf("warning: tcp service backend auto-heal failed: %v", err)
		return
	}
	if len(healed) == 0 {
		return
	}
	sort.Strings(healed)
	detail := fmt.Sprintf("restored tcp service backends: %s", joinLimited(healed, 8))
	o.log.Printf("%s", detail)
	o.noteStartupAutoHeal(detail)
}

// healFailedTCPServiceBackends runs one orchestration or CLI step.
// Signature: (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error).
// Why: failed TCP protocol checks need a direct backend-heal primitive that can
// be reused by startup waits and post-start daemon repair.
func (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error) {
	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
		return nil, nil
	}
	healed := []string{}
	attempted := map[string]struct{}{}
	for _, check := range o.cfg.Startup.TCPServiceChecklist {
		namespace := strings.TrimSpace(check.Namespace)
		service := strings.TrimSpace(check.Service)
		if namespace == "" || service == "" {
			continue
		}
		key := namespace + "/" + service
		if _, ok := attempted[key]; ok {
			continue
		}
		ok, _ := o.tcpServiceCheckReady(ctx, check)
		if ok {
			continue
		}
		attempted[key] = struct{}{}
		items, err := o.maybeHealCriticalEndpointBackends(ctx, namespace, service)
		if err != nil {
			return healed, fmt.Errorf("%s/%s: %w", namespace, service, err)
		}
		healed = append(healed, items...)
	}
	return healed, nil
}