package service import ( "encoding/json" "net/http" "net/http/httptest" "strings" "testing" "time" "metis/pkg/inventory" ) func TestRemoteHelperBranches(t *testing.T) { if got := prettyDeviceTarget(""); got != "the selected target" { t.Fatalf("prettyDeviceTarget empty = %q", got) } if got := prettyDeviceTarget("hosttmp:///var/tmp/metis-flash-test"); got != "/var/tmp/metis-flash-test" { t.Fatalf("prettyDeviceTarget hosttmp = %q", got) } if got := ramp(0, 10, 20, 1, 2); got != 1 { t.Fatalf("ramp before start = %v", got) } if got := ramp(0, 1, 1, 2, 3); got != 3 { t.Fatalf("ramp collapsed range = %v", got) } if got := ramp(20, 10, 20, 1, 2); got != 2 { t.Fatalf("ramp after end = %v", got) } if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp" { t.Fatalf("mountedHostTmpDir = %q", got) } if got := mountedHostTmpDir("/var/tmp/metis-flash-test"); got != "/host-tmp" { t.Fatalf("mountedHostTmpDir non-tmp = %q", got) } if got := shellQuote(""); got != "''" { t.Fatalf("shellQuote empty = %q", got) } if got := shellQuote("a'b"); got != `'a'"'"'b'` { t.Fatalf("shellQuote = %q", got) } if got, msg := buildStageHeartbeat("n1", "b1", 5*time.Second); got < 8 || !strings.Contains(msg, "Scheduling") { t.Fatalf("buildStageHeartbeat early = %v %q", got, msg) } if got, msg := buildStageHeartbeat("n1", "b1", 400*time.Second); got < 58 || got > 70 || !strings.Contains(msg, "Compressing") { t.Fatalf("buildStageHeartbeat compress = %v %q", got, msg) } if got, msg := flashStageHeartbeat("h1", "artifact", 15*time.Second); got < 88 || !strings.Contains(msg, "Writing") { t.Fatalf("flashStageHeartbeat = %v %q", got, msg) } app := newTestApp(t) app.settings.HarborRegistry = "registry.example" app.settings.HarborProject = "metis" app.artifactStore["n1"] = ArtifactSummary{Node: "n1", Ref: "registry.example/metis/n1:latest"} if got := app.remoteArtifactNote("n1"); got != "registry.example/metis/n1:latest" { t.Fatalf("remoteArtifactNote = %q", got) } if got := app.remoteArtifactNote("n2"); got != "registry.example/metis/n2:latest" { t.Fatalf("remoteArtifactNote fallback = %q", got) } if _, err := app.ensureDevice("titan-22", ""); err == nil { t.Fatal("expected empty device selection to fail") } if got := inventoryNodeArch(&inventory.NodeSpec{}, &inventory.NodeClass{Arch: "amd64"}); got != "amd64" { t.Fatalf("inventoryNodeArch = %q", got) } worker := remoteWorkerEntrypoint(true, "--node", "n1") if !strings.Contains(worker, "metis-runtime-env.sh") || !strings.Contains(worker, "metis-ssh-env.sh") { t.Fatalf("remoteWorkerEntrypoint missing expected sources: %s", worker) } } func TestSelectBuilderHostPrefersWorkerAndArch(t *testing.T) { kube := fakeKubeServer(t) installKubeFactory(t, kube) app := newTestApp(t) node, err := app.selectBuilderHost("arm64", "titan-22") if err != nil { t.Fatalf("selectBuilderHost: %v", err) } if node.Name != "titan-22" { t.Fatalf("expected titan-22 builder, got %s", node.Name) } } func TestSelectBuilderHostErrorBranch(t *testing.T) { kube := fakeKubeServer(t) installKubeFactory(t, kube) app := newTestApp(t) if _, err := app.selectBuilderHost("s390x", "titan-22"); err == nil { t.Fatal("expected selectBuilderHost error") } } func TestSelectBuilderHostAvoidsBusyBuilderWhenPeersAreFree(t *testing.T) { kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes": _ = json.NewEncoder(w).Encode(map[string]any{ "items": []any{ map[string]any{ "metadata": map[string]any{ "name": "titan-04", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "titan-05", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, }, }) case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": _ = json.NewEncoder(w).Encode(map[string]any{ "items": []any{ map[string]any{ "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, "spec": map[string]any{"nodeName": "titan-04"}, "status": map[string]any{"phase": "Running"}, }, }, }) default: http.NotFound(w, r) } })) defer kube.Close() installKubeFactory(t, kube) app := newTestApp(t) app.settings.Namespace = "maintenance" node, err := app.selectBuilderHost("arm64", "") if err != nil { t.Fatalf("selectBuilderHost: %v", err) } if node.Name != "titan-05" { t.Fatalf("expected titan-05 builder, got %s", node.Name) } } func TestEnsureDeviceRejectsUnknownCandidate(t *testing.T) { kube := fakeKubeServer(t) installKubeFactory(t, kube) app := newTestApp(t) app.settings.Namespace = "maintenance" app.settings.RunnerImageARM64 = "runner:arm64" if _, err := app.ensureDevice("titan-22", "/dev/sda"); err == nil { t.Fatal("expected stale device selection to fail") } } func TestSelectBuilderHostScoresStorageAndAMD64(t *testing.T) { kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes": _ = json.NewEncoder(w).Encode(map[string]any{ "items": []any{ map[string]any{ "metadata": map[string]any{ "name": "storage-arm", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "free-arm", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "desktop", "labels": map[string]string{ "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "titan-24", "labels": map[string]string{ "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, }, }) case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}}) default: http.NotFound(w, r) } })) defer kube.Close() installKubeFactory(t, kube) app := newTestApp(t) app.settings.Namespace = "maintenance" app.settings.DefaultFlashHost = "desktop" app.inventory.Nodes = []inventory.NodeSpec{ { Name: "storage-arm", LonghornDisks: []inventory.LonghornDisk{{Mountpoint: "/var/lib/longhorn"}}, }, } node, err := app.selectBuilderHost("arm64", "") if err != nil { t.Fatalf("selectBuilderHost arm64: %v", err) } if node.Name != "free-arm" { t.Fatalf("expected free-arm to beat storage node, got %s", node.Name) } node, err = app.selectBuilderHost("amd64", "") if err != nil { t.Fatalf("selectBuilderHost amd64: %v", err) } if node.Name != "desktop" { t.Fatalf("expected desktop to beat titan-24, got %s", node.Name) } } func TestRemoteWorkspaceAndHostTmpPathsPreferUsbScratchRoots(t *testing.T) { app := newTestApp(t) app.settings.RemoteWorkspaceDir = "/var/tmp/metis-workspace" app.settings.HostTmpDir = "/var/tmp/metis-flash-test" app.desiredMetadata["titan-10"] = DesiredNodeMetadata{ Node: "titan-10", Labels: map[string]string{"hardware": "rpi5"}, Taints: []string{"dedicated=recovery:NoSchedule"}, } buildSpec := app.remoteBuildPodSpec("metis-build-123", "titan-04", "runner:arm64", "titan-10", "titan-10", "registry.example/metis/titan-10", "build-1") buildBody := buildSpec["spec"].(map[string]any) buildVolumes := buildBody["volumes"].([]map[string]any) workspaceVolume := buildVolumes[0]["hostPath"].(map[string]any) if got := workspaceVolume["path"]; got != "/var/tmp/metis-workspace/metis-build-123" { t.Fatalf("build workspace hostPath = %v", got) } buildContainer := buildBody["containers"].([]map[string]any)[0] buildEnv := buildContainer["env"].([]map[string]any) if len(buildEnv) != 2 { t.Fatalf("expected desired metadata env, got %#v", buildEnv) } metadataAnnotations := buildSpec["metadata"].(map[string]any)["annotations"].(map[string]string) if metadataAnnotations["vault.hashicorp.com/agent-inject-secret-metis-node-secrets-env.sh"] != "secret/data/nodes/titan-10" { t.Fatalf("unexpected node secret annotation: %#v", metadataAnnotations) } if !strings.Contains(metadataAnnotations["vault.hashicorp.com/agent-inject-template-metis-node-secrets-env.sh"], "METIS_NODE_ROOT_PASSWORD") { t.Fatalf("expected node password exports in vault template: %#v", metadataAnnotations) } buildSecurity := buildContainer["securityContext"].(map[string]any) if got := buildSecurity["runAsUser"]; got != 0 { t.Fatalf("build runAsUser = %v", got) } if got := buildSecurity["runAsGroup"]; got != 0 { t.Fatalf("build runAsGroup = %v", got) } flashSpec := app.remoteFlashPodSpec("metis-flash-123", "titan-04", "runner:arm64", "titan-10", hostTmpDevicePath, "registry.example/metis/titan-10") flashVolumes := flashSpec["spec"].(map[string]any)["volumes"].([]map[string]any) flashWorkspace := flashVolumes[0]["hostPath"].(map[string]any) if got := flashWorkspace["path"]; got != "/var/tmp/metis-workspace/metis-flash-123" { t.Fatalf("flash workspace hostPath = %v", got) } hostTmp := flashVolumes[4]["hostPath"].(map[string]any) if got := hostTmp["path"]; got != "/var/tmp/metis-flash-test" { t.Fatalf("host tmp hostPath = %v", got) } } func TestSelectBuilderHostPrefersUsbScratchReadyWorkers(t *testing.T) { kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes": _ = json.NewEncoder(w).Encode(map[string]any{ "items": []any{ map[string]any{ "metadata": map[string]any{ "name": "titan-04", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, "annotations": map[string]string{ "maintenance.bstein.dev/usb-scratch-status": "ok", "maintenance.bstein.dev/usb-scratch-managed-paths": "/var/log/pods_/var/tmp", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "titan-05", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, }, }) case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}}) default: http.NotFound(w, r) } })) defer kube.Close() installKubeFactory(t, kube) app := newTestApp(t) app.settings.Namespace = "maintenance" node, err := app.selectBuilderHost("arm64", "") if err != nil { t.Fatalf("selectBuilderHost: %v", err) } if node.Name != "titan-04" { t.Fatalf("expected titan-04 scratch-ready builder, got %s", node.Name) } } func TestSelectBuilderHostTieBreaksByName(t *testing.T) { kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes": _ = json.NewEncoder(w).Encode(map[string]any{ "items": []any{ map[string]any{ "metadata": map[string]any{ "name": "beta", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, map[string]any{ "metadata": map[string]any{ "name": "alpha", "labels": map[string]string{ "kubernetes.io/arch": "arm64", "hardware": "rpi5", "node-role.kubernetes.io/worker": "true", }, }, "spec": map[string]any{"unschedulable": false}, }, }, }) case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}}) default: http.NotFound(w, r) } })) defer kube.Close() installKubeFactory(t, kube) app := newTestApp(t) app.settings.Namespace = "maintenance" app.inventory.Nodes = nil node, err := app.selectBuilderHost("arm64", "") if err != nil { t.Fatalf("selectBuilderHost: %v", err) } if node.Name != "alpha" { t.Fatalf("expected alphabetical tie-breaker, got %s", node.Name) } } func TestFlashPhaseHeartbeatAndManagedPathHelpers(t *testing.T) { cases := []struct { stage string minimum float64 phrase string }{ {stage: "flash_pull", minimum: 84, phrase: "Pulling"}, {stage: "flash_prepare", minimum: 88, phrase: "Preparing"}, {stage: "flash_unpack", minimum: 90, phrase: "Preparing"}, {stage: "flash_flush", minimum: 98, phrase: "Flushing"}, {stage: "flash_verify", minimum: 99, phrase: "Verifying"}, {stage: "flash", minimum: 90, phrase: "Writing"}, } for _, tc := range cases { got, msg := flashStagePhaseHeartbeat(tc.stage, "titan-22", "registry.example/metis/titan-15:latest", 20*time.Second) if got < tc.minimum || !strings.Contains(msg, tc.phrase) { t.Fatalf("flashStagePhaseHeartbeat(%s) = %v %q", tc.stage, got, msg) } } if !managedPathsContain("/var/log/pods_/var/tmp", "/var/tmp") { t.Fatal("expected managedPathsContain to match child paths") } if managedPathsContain("/var/log/pods", "/home/atlas") { t.Fatal("managedPathsContain should reject unrelated paths") } }