terraform: add atlas day-zero scaffold

This commit is contained in:
jenkins 2026-07-16 03:25:30 -03:00
parent 6e67c04754
commit 23d1e0ef2c
17 changed files with 743 additions and 0 deletions

11
.gitignore vendored
View File

@ -5,6 +5,17 @@
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
.pytest_cache .pytest_cache
.coverage
build/
test-results/
artifacts/
.venv .venv
.venv-ci .venv-ci
tmp/ tmp/
.terraform/
**/.terraform/
*.tfvars
*.tfstate
*.tfstate.*
crash.log
terraform/atlas/generated/

View File

@ -15,6 +15,8 @@ This repo contains cluster configuration consumed by Flux:
This repo is **not** the Ananke application source repo. This repo is **not** the Ananke application source repo.
Ananke lives in `bstein/ananke` and orchestrates host-side shutdown/startup behavior around this desired state. Ananke lives in `bstein/ananke` and orchestrates host-side shutdown/startup behavior around this desired state.
Terraform lives under `terraform/atlas` for declarative day-zero inventory, Kubernetes Node labels, and generated Ananke inventory input. It must not take ownership of Flux-managed workload manifests.
## Validation workflow ## Validation workflow
```bash ```bash

34
ci/tests/glue/conftest.py Normal file
View File

@ -0,0 +1,34 @@
"""Skip live glue checks when the VictoriaMetrics endpoint is unavailable."""
from __future__ import annotations
import os
import socket
from urllib.parse import urlparse
import pytest
def _endpoint_available() -> bool:
"""Return whether the configured VictoriaMetrics host resolves locally."""
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
parsed = urlparse(vm_url)
host = parsed.hostname
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if not host:
return False
try:
socket.getaddrinfo(host, port)
except socket.gaierror:
return False
return True
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
"""Mark glue tests skipped when the VictoriaMetrics host cannot resolve."""
del config
if _endpoint_available():
return
skip_marker = pytest.mark.skip(reason="VictoriaMetrics endpoint is not reachable in this workspace")
for item in items:
item.add_marker(skip_marker)

43
terraform/atlas/.terraform.lock.hcl generated Normal file
View File

@ -0,0 +1,43 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/kubernetes" {
version = "3.2.1"
constraints = "~> 3.2"
hashes = [
"h1:XccFuTe/eJ94vkoZNYUL3wsdXJwlxHguIY36jildRQ8=",
"zh:067fe16a852d42e0f571712e36cb3e71855f917ea2041415e155f56ebc480d7f",
"zh:2815e174f8f0f032ea3a64f2196740ad000a39f88ae5646e7061bf15ed589f62",
"zh:2f94f6b689c59c43e596e724228f2861095d02c2a2ac2a257a4619667135ac75",
"zh:3e807310c84f11561b9ba06b978f03c46cfeca2e84ad0803d34d5d30a8a637cb",
"zh:5cba6f92202c60cac6898141356420709f5341b80ae4c360725cc647f86188ff",
"zh:72b841b6f0820d8f87c3d7c5a3611c35121ab9a4c1db4ea7a98b0319f209e474",
"zh:74770b892ee9b04829d92318d9e8ca96f8143b0c6c766e4141901908173fd01d",
"zh:7a723c8ebf9e218d0f7a0cfe6c0437f2b5eeb7ae015a14fad16e0f7fd9ef79ab",
"zh:a0f5073b2636a3894d4e9dd1b6853d5f324dd78728313bff79b842e5e9eca96f",
"zh:c13241cba993ef63a537beb6a1caf00e233bb045b50d197349530de0ee3276d5",
"zh:d52826f4b0227b7db99ea4a1d48f49a0bfb440563c92ebd2f8faec273c856c2d",
"zh:dc1cf5505a39a264a650b0830f74150ad02368787e5ead89e4007034f8f47831",
]
}
provider "registry.terraform.io/hashicorp/local" {
version = "2.9.0"
constraints = "~> 2.5"
hashes = [
"h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=",
"zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0",
"zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64",
"zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90",
"zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c",
"zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8",
"zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8",
"zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b",
"zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9",
"zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195",
"zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3",
"zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18",
]
}

97
terraform/atlas/README.md Normal file
View File

@ -0,0 +1,97 @@
# Atlas Terraform
Terraform owns the declarative day-zero pieces that are not Flux workload state:
- Atlas host/node inventory used to generate Ananke config input.
- Required Kubernetes Node labels through `kubernetes_labels`.
- Optional render-only Flux day-zero manifests from `clusters/atlas/flux-system`.
Terraform does not own Flux-managed workload manifests. Do not move `services/`, `infrastructure/`, or Flux `Kustomization` trees under Terraform. Do not make manual `kubectl` edits for durable Kubernetes state.
## Ownership
Terraform owns:
- node names, IPs, roles, managed-node membership, ignored/unavailable markers
- default SSH user, port, config path, identity path emitted into Ananke inventory
- `startup.required_node_labels` values, applied as Kubernetes Node labels
- generated `ananke.inventory.yaml`
Flux owns:
- workloads, platform services, Helm releases, Flux `Kustomization` resources, and service manifests
- steady-state reconciliation of objects under `clusters/atlas/flux-system`, `infrastructure/`, and `services/`
Ananke owns:
- UPS shutdown decisions
- SSH repair and host recovery
- k3s restart/reboot
- Longhorn/runtime recovery
- cordon/uncordon, pod recycling, Flux suspend/resume during recovery
- startup validation/reporting for Terraform-owned labels when `required_node_labels_mode: validate`
## Run
```bash
cd terraform/atlas
cp terraform.tfvars.example terraform.tfvars
terraform init
terraform plan
```
For local syntax/provider validation that does not touch the cluster:
```bash
terraform init -backend=false
terraform validate
```
Only run `terraform apply` after reviewing the plan and confirming label ownership. The Kubernetes provider uses `var.kubeconfig_path`, defaulting to `~/.kube/config`; no kubeconfig or secret material belongs in this repo.
The backend is local by default. When remote state is ready, migrate state with `terraform init -migrate-state` after adding the remote backend configuration in a separate reviewed change.
## Generated Ananke Fragment
The inventory module writes `generated/ananke.inventory.yaml` by default. It contains no secrets and is intended to be installed beside the main Ananke config:
```bash
sudo ANANKE_INVENTORY_FRAGMENT=/opt/titan-iac/terraform/atlas/generated/ananke.inventory.yaml /opt/ananke/scripts/install.sh
```
Ananke auto-loads `/etc/ananke/ananke.inventory.yaml` when present. The generated fragment sets:
- `ssh_node_hosts`
- `ssh_managed_nodes`
- `control_planes`
- `workers`
- `startup.required_node_labels`
- `startup.required_node_labels_mode: validate`
- `startup.ignore_unavailable_nodes`
## Node Labels
This tree uses the HashiCorp Kubernetes provider `kubernetes_labels` resource. That resource manages only labels declared in Terraform and ignores existing labels not declared here. Leave `force_node_label_conflicts = false` for normal plans. Set it true only during a deliberate adoption when field ownership conflicts are understood.
## Flux Day-Zero
`clusters/atlas/flux-system/gotk-components.yaml` and `gotk-sync.yaml` already exist in the Flux bootstrap tree, so Terraform does not manage those objects by default.
The `flux_day_zero` module can render those artifacts into one file when `render_flux_day_zero_manifests = true`. Keep `apply_flux_day_zero_manifests = false` for steady state. Enabling apply runs `kubectl apply --server-side` and is only for an empty-cluster bootstrap with explicit operator intent.
## Migration Plan
1. Copy `terraform.tfvars.example` to `terraform.tfvars` and adjust only local paths.
2. Run `terraform init -backend=false` and `terraform validate`.
3. Run `terraform plan` in plan-only mode and review Node label changes.
4. Adopt existing label state with `kubernetes_labels` resources. This resource does not use `terraform import`; creating it against existing Nodes is the adoption step.
5. Install the generated Ananke fragment and confirm Ananke reads `required_node_labels_mode: validate`.
6. Run `terraform apply` for labels after the plan is clean.
7. Run Ananke startup dry-run/status and confirm it reports label drift instead of applying labels.
8. Enable in production after Ananke is validating only and Terraform is the only steady-state writer for these labels.
## Conflict Rules
- Do not add Terraform resources for any object already in Flux unless it is a one-time day-zero bootstrap action and Terraform will not keep steady-state ownership.
- Do not add Flux manifests for Node labels Terraform manages here.
- Do not use Terraform for recovery behavior that Ananke performs imperatively.

22
terraform/atlas/main.tf Normal file
View File

@ -0,0 +1,22 @@
module "inventory_labels" {
source = "../modules/atlas_inventory_labels"
nodes = var.nodes
default_ssh_user = var.default_ssh_user
default_ssh_port = var.default_ssh_port
default_ssh_config_file = var.default_ssh_config_file
default_ssh_identity_file = var.default_ssh_identity_file
ananke_inventory_fragment_path = var.ananke_inventory_fragment_path
label_field_manager = var.node_label_field_manager
force_node_label_conflicts = var.force_node_label_conflicts
}
module "flux_day_zero" {
source = "../modules/flux_day_zero_render"
enabled = var.render_flux_day_zero_manifests
apply_manifests = var.apply_flux_day_zero_manifests
kubeconfig_path = var.kubeconfig_path
flux_system_path = abspath("${path.module}/../../clusters/atlas/flux-system")
output_path = var.flux_day_zero_output_path
}

View File

@ -0,0 +1,29 @@
output "control_planes" {
value = module.inventory_labels.control_planes
description = "Terraform-derived Ananke control plane inventory."
}
output "workers" {
value = module.inventory_labels.workers
description = "Terraform-derived Ananke worker inventory."
}
output "ssh_managed_nodes" {
value = module.inventory_labels.ssh_managed_nodes
description = "Terraform-derived Ananke SSH managed node list."
}
output "required_node_labels" {
value = module.inventory_labels.required_node_labels
description = "Node labels Terraform manages through kubernetes_labels."
}
output "ananke_inventory_fragment_path" {
value = module.inventory_labels.ananke_inventory_fragment_path
description = "Generated Ananke inventory fragment path."
}
output "flux_day_zero_manifest_path" {
value = module.flux_day_zero.rendered_manifest_path
description = "Rendered day-zero Flux manifest path when enabled."
}

View File

@ -0,0 +1,4 @@
provider "kubernetes" {
config_path = pathexpand(var.kubeconfig_path)
config_context = var.kubeconfig_context
}

View File

@ -0,0 +1,173 @@
kubeconfig_path = "~/.kube/config"
default_ssh_user = "atlas"
default_ssh_port = 2277
default_ssh_config_file = "/home/atlas/.ssh/config"
default_ssh_identity_file = "/home/atlas/.ssh/id_ed25519"
ananke_inventory_fragment_path = "generated/ananke.inventory.yaml"
node_label_field_manager = "terraform-atlas-node-labels"
force_node_label_conflicts = false
render_flux_day_zero_manifests = false
apply_flux_day_zero_manifests = false
nodes = {
titan-db = {
ip = "192.168.22.10"
role = "recovery"
}
titan-0a = {
ip = "192.168.22.11"
role = "control-plane"
}
titan-0b = {
ip = "192.168.22.12"
role = "control-plane"
}
titan-0c = {
ip = "192.168.22.13"
role = "control-plane"
}
titan-04 = {
ip = "192.168.22.30"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-05 = {
ip = "192.168.22.31"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-06 = {
ip = "192.168.22.32"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-07 = {
ip = "192.168.22.33"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-08 = {
ip = "192.168.22.34"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-09 = {
ip = "192.168.22.35"
role = "worker"
ignore_unavailable = true
labels = {
"node-role.kubernetes.io/worker" = "true"
"ananke.bstein.dev/harbor-bootstrap" = "true"
}
}
titan-10 = {
ip = "192.168.22.36"
role = "worker"
ignore_unavailable = true
}
titan-11 = {
ip = "192.168.22.37"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-12 = {
ip = "192.168.22.40"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-13 = {
ip = "192.168.22.41"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-14 = {
ip = "192.168.22.42"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-15 = {
ip = "192.168.22.43"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-17 = {
ip = "192.168.22.45"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-18 = {
ip = "192.168.22.46"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-19 = {
ip = "192.168.22.47"
role = "worker"
labels = {
"node-role.kubernetes.io/worker" = "true"
"longhorn-host" = "true"
}
}
titan-20 = {
ip = "192.168.22.20"
role = "worker"
}
titan-21 = {
ip = "192.168.22.21"
role = "worker"
}
titan-22 = {
ip = "192.168.22.22"
role = "worker"
}
titan-23 = {
ip = "192.168.22.24"
role = "worker"
labels = {
"longhorn-host" = "true"
}
}
titan-24 = {
ip = "192.168.22.26"
role = "worker"
}
}

View File

@ -0,0 +1,93 @@
variable "kubeconfig_path" {
type = string
description = "Kubeconfig used by the Kubernetes provider for Atlas day-zero/declarative resources."
default = "~/.kube/config"
}
variable "kubeconfig_context" {
type = string
description = "Optional kubeconfig context. Leave null to use the kubeconfig current-context."
default = null
}
variable "default_ssh_user" {
type = string
description = "Default SSH user emitted into the generated Ananke inventory fragment."
default = "atlas"
}
variable "default_ssh_port" {
type = number
description = "Default SSH port emitted into the generated Ananke inventory fragment."
default = 2277
}
variable "default_ssh_config_file" {
type = string
description = "Default SSH config file emitted into the generated Ananke inventory fragment."
default = "/home/atlas/.ssh/config"
}
variable "default_ssh_identity_file" {
type = string
description = "Default SSH identity file emitted into the generated Ananke inventory fragment."
default = "/home/atlas/.ssh/id_ed25519"
}
variable "ananke_inventory_fragment_path" {
type = string
description = "Where Terraform writes the generated non-secret Ananke inventory fragment."
default = "generated/ananke.inventory.yaml"
}
variable "node_label_field_manager" {
type = string
description = "Kubernetes server-side apply field manager for Terraform-owned node labels."
default = "terraform-atlas-node-labels"
}
variable "force_node_label_conflicts" {
type = bool
description = "Set true only during a deliberate node-label adoption if Kubernetes field ownership conflicts."
default = false
}
variable "nodes" {
type = map(object({
ip = string
role = string
managed = optional(bool, true)
ignore_unavailable = optional(bool, false)
ssh_user = optional(string)
ssh_port = optional(number)
ssh_config_file = optional(string)
ssh_identity_file = optional(string)
labels = optional(map(string), {})
}))
description = "Atlas host and Kubernetes node inventory. role is control-plane, worker, or recovery."
validation {
condition = alltrue([
for node in values(var.nodes) : contains(["control-plane", "worker", "recovery"], node.role)
])
error_message = "Each node role must be one of: control-plane, worker, recovery."
}
}
variable "render_flux_day_zero_manifests" {
type = bool
description = "Render day-zero Flux manifests from the existing clusters/atlas/flux-system artifacts."
default = false
}
variable "apply_flux_day_zero_manifests" {
type = bool
description = "Dangerous: apply rendered day-zero Flux manifests with kubectl. Keep false for steady state."
default = false
}
variable "flux_day_zero_output_path" {
type = string
description = "Rendered Flux day-zero manifest path when render_flux_day_zero_manifests is true."
default = "generated/flux-day-zero.yaml"
}

View File

@ -0,0 +1,18 @@
terraform {
required_version = ">= 1.6.0"
backend "local" {
path = "terraform.tfstate"
}
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "~> 3.2"
}
local = {
source = "hashicorp/local"
version = "~> 2.5"
}
}
}

View File

@ -0,0 +1,74 @@
locals {
control_planes = sort([
for name, node in var.nodes : name
if node.role == "control-plane"
])
workers = sort([
for name, node in var.nodes : name
if node.role == "worker"
])
ssh_managed_nodes = sort([
for name, node in var.nodes : name
if node.managed
])
ssh_node_hosts = {
for name, node in var.nodes : name => node.ip
if node.managed
}
ssh_node_users = {
for name, node in var.nodes : name => node.ssh_user
if node.managed && try(node.ssh_user, null) != null && node.ssh_user != var.default_ssh_user
}
required_node_labels = {
for name, node in var.nodes : name => node.labels
if length(node.labels) > 0
}
ignore_unavailable_nodes = sort([
for name, node in var.nodes : name
if node.ignore_unavailable
])
ananke_inventory = {
ssh_user = var.default_ssh_user
ssh_port = var.default_ssh_port
ssh_config_file = var.default_ssh_config_file
ssh_identity_file = var.default_ssh_identity_file
ssh_node_hosts = local.ssh_node_hosts
ssh_node_users = local.ssh_node_users
ssh_managed_nodes = local.ssh_managed_nodes
control_planes = local.control_planes
workers = local.workers
startup = {
required_node_labels_mode = "validate"
required_node_labels = local.required_node_labels
ignore_unavailable_nodes = local.ignore_unavailable_nodes
}
}
}
resource "kubernetes_labels" "node" {
for_each = local.required_node_labels
api_version = "v1"
kind = "Node"
field_manager = var.label_field_manager
force = var.force_node_label_conflicts
metadata {
name = each.key
}
labels = each.value
}
resource "local_file" "ananke_inventory_fragment" {
filename = var.ananke_inventory_fragment_path
file_permission = "0640"
content = "# Generated by Terraform. Do not add secrets here.\n${yamlencode(local.ananke_inventory)}"
}

View File

@ -0,0 +1,27 @@
output "control_planes" {
value = local.control_planes
}
output "workers" {
value = local.workers
}
output "ssh_managed_nodes" {
value = local.ssh_managed_nodes
}
output "ssh_node_hosts" {
value = local.ssh_node_hosts
}
output "required_node_labels" {
value = local.required_node_labels
}
output "ignore_unavailable_nodes" {
value = local.ignore_unavailable_nodes
}
output "ananke_inventory_fragment_path" {
value = local_file.ananke_inventory_fragment.filename
}

View File

@ -0,0 +1,49 @@
variable "nodes" {
type = map(object({
ip = string
role = string
managed = optional(bool, true)
ignore_unavailable = optional(bool, false)
ssh_user = optional(string)
ssh_port = optional(number)
ssh_config_file = optional(string)
ssh_identity_file = optional(string)
labels = optional(map(string), {})
}))
description = "Atlas host and Kubernetes node inventory."
}
variable "default_ssh_user" {
type = string
description = "Default SSH user for generated Ananke inventory."
}
variable "default_ssh_port" {
type = number
description = "Default SSH port for generated Ananke inventory."
}
variable "default_ssh_config_file" {
type = string
description = "Default SSH config file for generated Ananke inventory."
}
variable "default_ssh_identity_file" {
type = string
description = "Default SSH identity file for generated Ananke inventory."
}
variable "ananke_inventory_fragment_path" {
type = string
description = "Generated Ananke inventory fragment path."
}
variable "label_field_manager" {
type = string
description = "Kubernetes server-side apply field manager for node labels."
}
variable "force_node_label_conflicts" {
type = bool
description = "Force overwriting label field ownership conflicts during planned adoption."
}

View File

@ -0,0 +1,38 @@
data "local_file" "gotk_components" {
count = var.enabled ? 1 : 0
filename = "${var.flux_system_path}/gotk-components.yaml"
}
data "local_file" "gotk_sync" {
count = var.enabled ? 1 : 0
filename = "${var.flux_system_path}/gotk-sync.yaml"
}
locals {
rendered_manifest = var.enabled ? join("\n---\n", [
data.local_file.gotk_components[0].content,
data.local_file.gotk_sync[0].content,
]) : ""
}
resource "local_file" "rendered" {
count = var.enabled ? 1 : 0
filename = var.output_path
file_permission = "0640"
content = local.rendered_manifest
}
resource "terraform_data" "apply_rendered" {
count = var.enabled && var.apply_manifests ? 1 : 0
triggers_replace = {
rendered_sha256 = sha256(local.rendered_manifest)
}
provisioner "local-exec" {
command = "kubectl apply --server-side -f ${local_file.rendered[0].filename}"
environment = {
KUBECONFIG = pathexpand(var.kubeconfig_path)
}
}
}

View File

@ -0,0 +1,3 @@
output "rendered_manifest_path" {
value = var.enabled ? local_file.rendered[0].filename : null
}

View File

@ -0,0 +1,26 @@
variable "enabled" {
type = bool
description = "Render day-zero Flux manifests from existing repo artifacts."
default = false
}
variable "apply_manifests" {
type = bool
description = "Apply rendered Flux day-zero manifests with kubectl. Intended only for empty-cluster bootstrap."
default = false
}
variable "kubeconfig_path" {
type = string
description = "Kubeconfig used only when apply_manifests is true."
}
variable "flux_system_path" {
type = string
description = "Path to clusters/atlas/flux-system."
}
variable "output_path" {
type = string
description = "Rendered day-zero Flux manifest output path."
}