134 lines
5.1 KiB
Go
134 lines
5.1 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log"
|
|
"regexp"
|
|
"sync"
|
|
"time"
|
|
|
|
"scm.bstein.dev/bstein/ananke/internal/config"
|
|
"scm.bstein.dev/bstein/ananke/internal/execx"
|
|
"scm.bstein.dev/bstein/ananke/internal/state"
|
|
)
|
|
|
|
type Orchestrator struct {
|
|
cfg config.Config
|
|
runner *execx.Runner
|
|
store *state.Store
|
|
log *log.Logger
|
|
runOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
|
|
runSensitiveOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
|
|
sshInputOverride func(timeoutCtx context.Context, timeout time.Duration, node string, command string, input string) (string, error)
|
|
startupReportMu sync.Mutex
|
|
activeStartupReport *startupReport
|
|
}
|
|
|
|
type commandOverrideFunc func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
|
|
|
|
type StartupOptions struct {
|
|
ForceFluxBranch string
|
|
SkipLocalBootstrap bool
|
|
Reason string
|
|
}
|
|
|
|
type ShutdownOptions struct {
|
|
SkipEtcdSnapshot bool
|
|
SkipDrain bool
|
|
Mode string
|
|
Reason string
|
|
}
|
|
|
|
type EtcdRestoreOptions struct {
|
|
ControlPlane string
|
|
SnapshotPath string
|
|
}
|
|
|
|
type startupWorkload struct {
|
|
Namespace string
|
|
Kind string
|
|
Name string
|
|
}
|
|
|
|
type workloadScaleEntry struct {
|
|
Namespace string `json:"namespace"`
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name"`
|
|
Replicas int `json:"replicas"`
|
|
}
|
|
|
|
type remotePeerStatus struct {
|
|
Intent state.Intent
|
|
BootstrapActive bool
|
|
}
|
|
|
|
type workloadScaleSnapshot struct {
|
|
GeneratedAt time.Time `json:"generated_at"`
|
|
Entries []workloadScaleEntry `json:"entries"`
|
|
}
|
|
|
|
type startupReport struct {
|
|
StartedAt time.Time `json:"started_at"`
|
|
Completed time.Time `json:"completed_at"`
|
|
Reason string `json:"reason"`
|
|
Status string `json:"status"`
|
|
Phase string `json:"phase"`
|
|
Success bool `json:"success"`
|
|
Error string `json:"error,omitempty"`
|
|
Checks map[string]startupCheckRecord `json:"checks"`
|
|
AutoHeals []string `json:"auto_heals"`
|
|
SourceHost string `json:"source_host"`
|
|
LastUpdated time.Time `json:"last_updated"`
|
|
}
|
|
|
|
type startupCheckRecord struct {
|
|
Status string `json:"status"`
|
|
Detail string `json:"detail"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
var datastoreEndpointPattern = regexp.MustCompile(`--datastore-endpoint(?:=|\s+)(?:'([^']+)'|"([^"]+)"|([^\s\\]+))`)
|
|
|
|
var criticalStartupWorkloads = []startupWorkload{
|
|
{Namespace: "flux-system", Kind: "deployment", Name: "source-controller"},
|
|
{Namespace: "flux-system", Kind: "deployment", Name: "kustomize-controller"},
|
|
{Namespace: "flux-system", Kind: "deployment", Name: "helm-controller"},
|
|
{Namespace: "flux-system", Kind: "deployment", Name: "notification-controller"},
|
|
{Namespace: "vault", Kind: "statefulset", Name: "vault"},
|
|
{Namespace: "postgres", Kind: "statefulset", Name: "postgres"},
|
|
{Namespace: "gitea", Kind: "deployment", Name: "gitea"},
|
|
{Namespace: "monitoring", Kind: "deployment", Name: "grafana"},
|
|
{Namespace: "monitoring", Kind: "statefulset", Name: "victoria-metrics-single-server"},
|
|
{Namespace: "monitoring", Kind: "deployment", Name: "kube-state-metrics"},
|
|
{Namespace: "logging", Kind: "deployment", Name: "oauth2-proxy-logs"},
|
|
{Namespace: "logging", Kind: "deployment", Name: "opensearch-dashboards"},
|
|
{Namespace: "logging", Kind: "statefulset", Name: "opensearch"},
|
|
}
|
|
|
|
var ErrEtcdRestoreNotApplicable = errors.New("etcd restore not applicable")
|
|
|
|
// New runs one orchestration or CLI step.
|
|
// Signature: New(cfg config.Config, runner *execx.Runner, store *state.Store, logger *log.Logger) *Orchestrator.
|
|
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
|
|
func New(cfg config.Config, runner *execx.Runner, store *state.Store, logger *log.Logger) *Orchestrator {
|
|
return &Orchestrator{cfg: cfg, runner: runner, store: store, log: logger}
|
|
}
|
|
|
|
// SetCommandOverrides runs one orchestration or CLI step.
|
|
// Signature: (o *Orchestrator) SetCommandOverrides(run commandOverrideFunc, runSensitive commandOverrideFunc).
|
|
// Why: enables deterministic integration testing from the top-level testing module
|
|
// without requiring package-local test files or live cluster dependencies.
|
|
func (o *Orchestrator) SetCommandOverrides(run commandOverrideFunc, runSensitive commandOverrideFunc) {
|
|
o.runOverride = run
|
|
o.runSensitiveOverride = runSensitive
|
|
}
|
|
|
|
// SetSSHInputOverride runs one orchestration or CLI step.
|
|
// Signature: (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)).
|
|
// Why: password-backed host-repair tests need to assert stdin behavior without
|
|
// exposing secrets through ordinary command-argument stubs.
|
|
func (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)) {
|
|
o.sshInputOverride = run
|
|
}
|