metis/pkg/service/remote_helpers_test.go

package service

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"metis/pkg/inventory"
)

func TestRemoteHelperBranches(t *testing.T) {
	if got := prettyDeviceTarget(""); got != "the selected target" {
		t.Fatalf("prettyDeviceTarget empty = %q", got)
	}
	if got := prettyDeviceTarget("hosttmp:///var/tmp/metis-flash-test"); got != "/var/tmp/metis-flash-test" {
		t.Fatalf("prettyDeviceTarget hosttmp = %q", got)
	}
	if got := ramp(0, 10, 20, 1, 2); got != 1 {
		t.Fatalf("ramp before start = %v", got)
	}
	if got := ramp(0, 1, 1, 2, 3); got != 3 {
		t.Fatalf("ramp collapsed range = %v", got)
	}
	if got := ramp(20, 10, 20, 1, 2); got != 2 {
		t.Fatalf("ramp after end = %v", got)
	}
	if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp" {
		t.Fatalf("mountedHostTmpDir = %q", got)
	}
	if got := mountedHostTmpDir("/var/tmp/metis-flash-test"); got != "/host-tmp" {
		t.Fatalf("mountedHostTmpDir non-tmp = %q", got)
	}
	if got := shellQuote(""); got != "''" {
		t.Fatalf("shellQuote empty = %q", got)
	}
	if got := shellQuote("a'b"); got != `'a'"'"'b'` {
		t.Fatalf("shellQuote = %q", got)
	}
	if got, msg := buildStageHeartbeat("n1", "b1", 5*time.Second); got < 8 || !strings.Contains(msg, "Scheduling") {
		t.Fatalf("buildStageHeartbeat early = %v %q", got, msg)
	}
	if got, msg := buildStageHeartbeat("n1", "b1", 400*time.Second); got < 58 || got > 70 || !strings.Contains(msg, "Compressing") {
		t.Fatalf("buildStageHeartbeat compress = %v %q", got, msg)
	}
	if got, msg := flashStageHeartbeat("h1", "artifact", 15*time.Second); got < 88 || !strings.Contains(msg, "Writing") {
		t.Fatalf("flashStageHeartbeat = %v %q", got, msg)
	}

	app := newTestApp(t)
	app.settings.HarborRegistry = "registry.example"
	app.settings.HarborProject = "metis"
	app.artifactStore["n1"] = ArtifactSummary{Node: "n1", Ref: "registry.example/metis/n1:latest"}
	if got := app.remoteArtifactNote("n1"); got != "registry.example/metis/n1:latest" {
		t.Fatalf("remoteArtifactNote = %q", got)
	}
	if got := app.remoteArtifactNote("n2"); got != "registry.example/metis/n2:latest" {
		t.Fatalf("remoteArtifactNote fallback = %q", got)
	}
	if _, err := app.ensureDevice("titan-22", ""); err == nil {
		t.Fatal("expected empty device selection to fail")
	}
	if got := inventoryNodeArch(&inventory.NodeSpec{}, &inventory.NodeClass{Arch: "amd64"}); got != "amd64" {
		t.Fatalf("inventoryNodeArch = %q", got)
	}
	worker := remoteWorkerEntrypoint(true, "--node", "n1")
	if !strings.Contains(worker, "metis-runtime-env.sh") || !strings.Contains(worker, "metis-ssh-env.sh") {
		t.Fatalf("remoteWorkerEntrypoint missing expected sources: %s", worker)
	}
}

func TestSelectBuilderHostPrefersWorkerAndArch(t *testing.T) {
	kube := fakeKubeServer(t)
	installKubeFactory(t, kube)
	app := newTestApp(t)
	node, err := app.selectBuilderHost("arm64", "titan-22")
	if err != nil {
		t.Fatalf("selectBuilderHost: %v", err)
	}
	if node.Name != "titan-22" {
		t.Fatalf("expected titan-22 builder, got %s", node.Name)
	}
}

func TestSelectBuilderHostErrorBranch(t *testing.T) {
	kube := fakeKubeServer(t)
	installKubeFactory(t, kube)
	app := newTestApp(t)
	if _, err := app.selectBuilderHost("s390x", "titan-22"); err == nil {
		t.Fatal("expected selectBuilderHost error")
	}
}

func TestSelectBuilderHostAvoidsBusyBuilderWhenPeersAreFree(t *testing.T) {
	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{
							"name": "titan-04",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "titan-05",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
				},
			})
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}},
						"spec":     map[string]any{"nodeName": "titan-04"},
						"status":   map[string]any{"phase": "Running"},
					},
				},
			})
		default:
			http.NotFound(w, r)
		}
	}))
	defer kube.Close()
	installKubeFactory(t, kube)
	app := newTestApp(t)
	app.settings.Namespace = "maintenance"
	node, err := app.selectBuilderHost("arm64", "")
	if err != nil {
		t.Fatalf("selectBuilderHost: %v", err)
	}
	if node.Name != "titan-05" {
		t.Fatalf("expected titan-05 builder, got %s", node.Name)
	}
}

func TestEnsureDeviceRejectsUnknownCandidate(t *testing.T) {
	kube := fakeKubeServer(t)
	installKubeFactory(t, kube)
	app := newTestApp(t)
	app.settings.Namespace = "maintenance"
	app.settings.RunnerImageARM64 = "runner:arm64"

	if _, err := app.ensureDevice("titan-22", "/dev/sda"); err == nil {
		t.Fatal("expected stale device selection to fail")
	}
}

func TestSelectBuilderHostScoresStorageAndAMD64(t *testing.T) {
	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{
							"name": "storage-arm",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "free-arm",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "desktop",
							"labels": map[string]string{
								"kubernetes.io/arch":             "amd64",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "titan-24",
							"labels": map[string]string{
								"kubernetes.io/arch":             "amd64",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
				},
			})
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods":
			_ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}})
		default:
			http.NotFound(w, r)
		}
	}))
	defer kube.Close()
	installKubeFactory(t, kube)
	app := newTestApp(t)
	app.settings.Namespace = "maintenance"
	app.settings.DefaultFlashHost = "desktop"
	app.inventory.Nodes = []inventory.NodeSpec{
		{
			Name:          "storage-arm",
			LonghornDisks: []inventory.LonghornDisk{{Mountpoint: "/var/lib/longhorn"}},
		},
	}

	node, err := app.selectBuilderHost("arm64", "")
	if err != nil {
		t.Fatalf("selectBuilderHost arm64: %v", err)
	}
	if node.Name != "free-arm" {
		t.Fatalf("expected free-arm to beat storage node, got %s", node.Name)
	}

	node, err = app.selectBuilderHost("amd64", "")
	if err != nil {
		t.Fatalf("selectBuilderHost amd64: %v", err)
	}
	if node.Name != "desktop" {
		t.Fatalf("expected desktop to beat titan-24, got %s", node.Name)
	}
}

func TestRemoteWorkspaceAndHostTmpPathsPreferUsbScratchRoots(t *testing.T) {
	app := newTestApp(t)
	app.settings.RemoteWorkspaceDir = "/var/tmp/metis-workspace"
	app.settings.HostTmpDir = "/var/tmp/metis-flash-test"
	app.desiredMetadata["titan-10"] = DesiredNodeMetadata{
		Node:   "titan-10",
		Labels: map[string]string{"hardware": "rpi5"},
		Taints: []string{"dedicated=recovery:NoSchedule"},
	}

	buildSpec := app.remoteBuildPodSpec("metis-build-123", "titan-04", "runner:arm64", "titan-10", "titan-10", "registry.example/metis/titan-10", "build-1")
	buildBody := buildSpec["spec"].(map[string]any)
	buildVolumes := buildBody["volumes"].([]map[string]any)
	workspaceVolume := buildVolumes[0]["hostPath"].(map[string]any)
	if got := workspaceVolume["path"]; got != "/var/tmp/metis-workspace/metis-build-123" {
		t.Fatalf("build workspace hostPath = %v", got)
	}
	buildContainer := buildBody["containers"].([]map[string]any)[0]
	buildEnv := buildContainer["env"].([]map[string]any)
	if len(buildEnv) != 2 {
		t.Fatalf("expected desired metadata env, got %#v", buildEnv)
	}
	metadataAnnotations := buildSpec["metadata"].(map[string]any)["annotations"].(map[string]string)
	if metadataAnnotations["vault.hashicorp.com/agent-inject-secret-metis-node-secrets-env.sh"] != "kv/data/atlas/nodes/titan-10" {
		t.Fatalf("unexpected node secret annotation: %#v", metadataAnnotations)
	}
	if !strings.Contains(metadataAnnotations["vault.hashicorp.com/agent-inject-template-metis-node-secrets-env.sh"], "METIS_NODE_ROOT_PASSWORD") {
		t.Fatalf("expected node password exports in vault template: %#v", metadataAnnotations)
	}
	buildSecurity := buildContainer["securityContext"].(map[string]any)
	if got := buildSecurity["runAsUser"]; got != 0 {
		t.Fatalf("build runAsUser = %v", got)
	}
	if got := buildSecurity["runAsGroup"]; got != 0 {
		t.Fatalf("build runAsGroup = %v", got)
	}

	flashSpec := app.remoteFlashPodSpec("metis-flash-123", "titan-04", "runner:arm64", "titan-10", hostTmpDevicePath, "registry.example/metis/titan-10")
	flashVolumes := flashSpec["spec"].(map[string]any)["volumes"].([]map[string]any)
	flashWorkspace := flashVolumes[0]["hostPath"].(map[string]any)
	if got := flashWorkspace["path"]; got != "/var/tmp/metis-workspace/metis-flash-123" {
		t.Fatalf("flash workspace hostPath = %v", got)
	}
	hostTmp := flashVolumes[4]["hostPath"].(map[string]any)
	if got := hostTmp["path"]; got != "/var/tmp/metis-flash-test" {
		t.Fatalf("host tmp hostPath = %v", got)
	}
}

func TestSelectBuilderHostPrefersUsbScratchReadyWorkers(t *testing.T) {
	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{
							"name": "titan-04",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
							"annotations": map[string]string{
								"maintenance.bstein.dev/usb-scratch-status":        "ok",
								"maintenance.bstein.dev/usb-scratch-managed-paths": "/var/log/pods_/var/tmp",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "titan-05",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
				},
			})
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods":
			_ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}})
		default:
			http.NotFound(w, r)
		}
	}))
	defer kube.Close()
	installKubeFactory(t, kube)
	app := newTestApp(t)
	app.settings.Namespace = "maintenance"

	node, err := app.selectBuilderHost("arm64", "")
	if err != nil {
		t.Fatalf("selectBuilderHost: %v", err)
	}
	if node.Name != "titan-04" {
		t.Fatalf("expected titan-04 scratch-ready builder, got %s", node.Name)
	}
}

func TestSelectBuilderHostTieBreaksByName(t *testing.T) {
	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{
							"name": "beta",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{
							"name": "alpha",
							"labels": map[string]string{
								"kubernetes.io/arch":             "arm64",
								"hardware":                       "rpi5",
								"node-role.kubernetes.io/worker": "true",
							},
						},
						"spec": map[string]any{"unschedulable": false},
					},
				},
			})
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods":
			_ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}})
		default:
			http.NotFound(w, r)
		}
	}))
	defer kube.Close()
	installKubeFactory(t, kube)
	app := newTestApp(t)
	app.settings.Namespace = "maintenance"
	app.inventory.Nodes = nil

	node, err := app.selectBuilderHost("arm64", "")
	if err != nil {
		t.Fatalf("selectBuilderHost: %v", err)
	}
	if node.Name != "alpha" {
		t.Fatalf("expected alphabetical tie-breaker, got %s", node.Name)
	}
}

func TestFlashPhaseHeartbeatAndManagedPathHelpers(t *testing.T) {
	cases := []struct {
		stage   string
		minimum float64
		phrase  string
	}{
		{stage: "flash_pull", minimum: 84, phrase: "Pulling"},
		{stage: "flash_prepare", minimum: 88, phrase: "Preparing"},
		{stage: "flash_unpack", minimum: 90, phrase: "Preparing"},
		{stage: "flash_flush", minimum: 98, phrase: "Flushing"},
		{stage: "flash_verify", minimum: 99, phrase: "Verifying"},
		{stage: "flash", minimum: 90, phrase: "Writing"},
	}
	for _, tc := range cases {
		got, msg := flashStagePhaseHeartbeat(tc.stage, "titan-22", "registry.example/metis/titan-15:latest", 20*time.Second)
		if got < tc.minimum || !strings.Contains(msg, tc.phrase) {
			t.Fatalf("flashStagePhaseHeartbeat(%s) = %v %q", tc.stage, got, msg)
		}
	}

	if !managedPathsContain("/var/log/pods_/var/tmp", "/var/tmp") {
		t.Fatal("expected managedPathsContain to match child paths")
	}
	if managedPathsContain("/var/log/pods", "/home/atlas") {
		t.Fatal("managedPathsContain should reject unrelated paths")
	}
}