diff --git a/pkg/service/cluster_remote_pod_test.go b/pkg/service/cluster_remote_pod_test.go new file mode 100644 index 0000000..a8d8555 --- /dev/null +++ b/pkg/service/cluster_remote_pod_test.go @@ -0,0 +1,269 @@ +package service + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestClusterActiveRemotePodLoadsCountsOnlyLivePods(t *testing.T) { + kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": + if got := r.URL.Query().Get("labelSelector"); got != "app=metis-remote,metis-run=build" { + t.Fatalf("unexpected labelSelector %q", got) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": []any{ + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": "titan-04"}, + "status": map[string]any{"phase": "Running"}, + }, + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": "titan-04"}, + "status": map[string]any{"phase": "Pending"}, + }, + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": "titan-05"}, + "status": map[string]any{"phase": "Succeeded"}, + }, + }, + }) + default: + http.NotFound(w, r) + } + })) + defer kube.Close() + + origFactory := kubeClientFactory + kubeClientFactory = func() (*kubeClient, error) { + return kubeClientFactoryForURL(kube.URL, kube.Client()), nil + } + t.Cleanup(func() { kubeClientFactory = origFactory }) + + loads := clusterActiveRemotePodLoads("maintenance", "build") + if loads["titan-04"] != 2 { + t.Fatalf("expected titan-04 load 2, got %#v", loads) + } + if _, ok := loads["titan-05"]; ok { + t.Fatalf("expected succeeded pod to be ignored, got %#v", loads) + } +} + +func TestClusterActiveRemotePodLoadsFiltersEdges(t *testing.T) { + kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": []any{ + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "flash"}}, + "spec": map[string]any{"nodeName": "wrong-run"}, + "status": map[string]any{"phase": "Running"}, + }, + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": ""}, + "status": map[string]any{"phase": "Running"}, + }, + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": "failed"}, + "status": map[string]any{"phase": "Failed"}, + }, + map[string]any{ + "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, + "spec": map[string]any{"nodeName": "good"}, + "status": map[string]any{"phase": "Pending"}, + }, + }, + }) + default: + http.NotFound(w, r) + } + })) + defer kube.Close() + + origFactory := kubeClientFactory + kubeClientFactory = func() (*kubeClient, error) { + return kubeClientFactoryForURL(kube.URL, kube.Client()), nil + } + t.Cleanup(func() { kubeClientFactory = origFactory }) + + loads := clusterActiveRemotePodLoads("maintenance", "build") + if len(loads) != 1 || loads["good"] != 1 { + t.Fatalf("unexpected filtered loads: %#v", loads) + } +} + +func TestRunRemotePodTerminalEdgeStates(t *testing.T) { + cases := []struct { + name string + phase string + reason string + message string + logStatus int + logBody string + want string + wantErr string + }{ + {name: "success uses logs when message empty", phase: "Succeeded", logStatus: http.StatusOK, logBody: "fallback payload", want: "fallback payload"}, + {name: "success reports missing payload and log error", phase: "Succeeded", logStatus: http.StatusInternalServerError, logBody: "no logs", wantErr: "logs unavailable"}, + {name: "success reports missing payload", phase: "Succeeded", logStatus: http.StatusOK, logBody: "", wantErr: "did not return a result payload"}, + {name: "failed uses logs when message empty", phase: "Failed", logStatus: http.StatusOK, logBody: "worker failed", wantErr: "worker failed"}, + {name: "failed falls back to default reason", phase: "Failed", logStatus: http.StatusOK, logBody: "", wantErr: "remote worker failed before reporting details"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + kube := remotePodStateServer(t, tc.phase, tc.reason, tc.message, tc.logStatus, tc.logBody) + installKubeFactory(t, kube) + app := newTestApp(t) + app.settings.Namespace = "maintenance" + + got, err := app.runRemotePod("", "metis-case", map[string]any{}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected error containing %q, got result=%q err=%v", tc.wantErr, got, err) + } + return + } + if err != nil || got != tc.want { + t.Fatalf("runRemotePod = %q err=%v", got, err) + } + }) + } +} + +func TestRunRemotePodProgressAndRequestFailures(t *testing.T) { + origFactory := kubeClientFactory + kubeClientFactory = func() (*kubeClient, error) { return nil, errors.New("offline") } + app := newTestApp(t) + if _, err := app.runRemotePod("", "metis-offline", map[string]any{}); err == nil { + t.Fatal("expected factory error") + } + kubeClientFactory = origFactory + + postFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + http.Error(w, "post denied", http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer postFail.Close() + installKubeFactory(t, postFail) + app = newTestApp(t) + app.settings.Namespace = "maintenance" + if _, err := app.runRemotePod("", "metis-post-fail", map[string]any{}); err == nil || !strings.Contains(err.Error(), "post denied") { + t.Fatalf("expected post failure, got %v", err) + } + + stateFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost: + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): + http.Error(w, "state denied", http.StatusForbidden) + default: + w.WriteHeader(http.StatusOK) + } + })) + defer stateFail.Close() + installKubeFactory(t, stateFail) + app = newTestApp(t) + app.settings.Namespace = "maintenance" + if _, err := app.runRemotePod("", "metis-state-fail", map[string]any{}); err == nil || !strings.Contains(err.Error(), "state denied") { + t.Fatalf("expected state failure, got %v", err) + } + + progressLog := ProgressLogLine(RemoteProgressUpdate{Stage: "flash", ProgressPct: 92, Message: "writing", WrittenBytes: 10, TotalBytes: 20}) + progressServer := remotePodStateServer(t, "Succeeded", "Completed", `{"ok":true}`, http.StatusOK, progressLog) + installKubeFactory(t, progressServer) + app = newTestApp(t) + app.settings.Namespace = "maintenance" + job := app.newJob("replace", "titan-15", "titan-22", "/dev/sdz") + app.setJob(job.ID, func(j *Job) { + j.Status = JobRunning + j.Stage = "build" + j.StageStartedAt = time.Now() + }) + if got, err := app.runRemotePod(job.ID, "metis-progress", map[string]any{}); err != nil || got != `{"ok":true}` { + t.Fatalf("runRemotePod progress = %q err=%v", got, err) + } + if got := app.job(job.ID); got == nil || got.Written != 10 || got.Total != 20 { + t.Fatalf("expected progress update, got %#v", got) + } +} + +func TestRemotePodStateAndLogErrorBranches(t *testing.T) { + kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/state-fail"): + http.Error(w, "state unavailable", http.StatusServiceUnavailable) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/log"): + http.Error(w, "plain log failure", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer kube.Close() + app := newTestApp(t) + app.settings.Namespace = "maintenance" + client := kubeClientFactoryForURL(kube.URL, kube.Client()) + + if _, err := app.remotePodState(client, "state-fail"); err == nil || !strings.Contains(err.Error(), "state unavailable") { + t.Fatalf("expected state error, got %v", err) + } + if _, err := app.remotePodLogs(client, "log-fail"); err == nil || !strings.Contains(err.Error(), "plain log failure") { + t.Fatalf("expected plain log error, got %v", err) + } + + badURL := &kubeClient{baseURL: "http://%zz", token: "tok", client: http.DefaultClient} + if _, err := app.remotePodLogs(badURL, "log-fail"); err == nil { + t.Fatal("expected log request creation error") + } + transportErr := &kubeClient{ + baseURL: "http://example.test", + token: "tok", + client: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("logs transport down") + })}, + } + if _, err := app.remotePodLogs(transportErr, "log-fail"); err == nil || !strings.Contains(err.Error(), "logs transport down") { + t.Fatalf("expected log transport error, got %v", err) + } +} + +func remotePodStateServer(t *testing.T, phase, reason, message string, logStatus int, logBody string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/pods"): + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/pods/"): + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/") && strings.HasSuffix(r.URL.Path, "/log"): + w.WriteHeader(logStatus) + _, _ = w.Write([]byte(logBody)) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "metadata": map[string]any{"name": filepath.Base(r.URL.Path)}, + "status": map[string]any{ + "phase": phase, + "reason": reason, + "message": message, + }, + }) + default: + http.NotFound(w, r) + } + })) +} diff --git a/pkg/service/cluster_test.go b/pkg/service/cluster_test.go index 2f85403..e599787 100644 --- a/pkg/service/cluster_test.go +++ b/pkg/service/cluster_test.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strings" "testing" - "time" ) type roundTripFunc func(*http.Request) (*http.Response, error) @@ -261,260 +260,3 @@ func TestDeleteNodeObjectFallback(t *testing.T) { t.Fatalf("deleteNodeObject fallback: %v", err) } } - -func TestClusterActiveRemotePodLoadsCountsOnlyLivePods(t *testing.T) { - kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": - if got := r.URL.Query().Get("labelSelector"); got != "app=metis-remote,metis-run=build" { - t.Fatalf("unexpected labelSelector %q", got) - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "items": []any{ - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": "titan-04"}, - "status": map[string]any{"phase": "Running"}, - }, - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": "titan-04"}, - "status": map[string]any{"phase": "Pending"}, - }, - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": "titan-05"}, - "status": map[string]any{"phase": "Succeeded"}, - }, - }, - }) - default: - http.NotFound(w, r) - } - })) - defer kube.Close() - - origFactory := kubeClientFactory - kubeClientFactory = func() (*kubeClient, error) { - return kubeClientFactoryForURL(kube.URL, kube.Client()), nil - } - t.Cleanup(func() { kubeClientFactory = origFactory }) - - loads := clusterActiveRemotePodLoads("maintenance", "build") - if loads["titan-04"] != 2 { - t.Fatalf("expected titan-04 load 2, got %#v", loads) - } - if _, ok := loads["titan-05"]; ok { - t.Fatalf("expected succeeded pod to be ignored, got %#v", loads) - } -} - -func TestClusterActiveRemotePodLoadsFiltersEdges(t *testing.T) { - kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods": - _ = json.NewEncoder(w).Encode(map[string]any{ - "items": []any{ - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "flash"}}, - "spec": map[string]any{"nodeName": "wrong-run"}, - "status": map[string]any{"phase": "Running"}, - }, - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": ""}, - "status": map[string]any{"phase": "Running"}, - }, - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": "failed"}, - "status": map[string]any{"phase": "Failed"}, - }, - map[string]any{ - "metadata": map[string]any{"labels": map[string]string{"app": "metis-remote", "metis-run": "build"}}, - "spec": map[string]any{"nodeName": "good"}, - "status": map[string]any{"phase": "Pending"}, - }, - }, - }) - default: - http.NotFound(w, r) - } - })) - defer kube.Close() - - origFactory := kubeClientFactory - kubeClientFactory = func() (*kubeClient, error) { - return kubeClientFactoryForURL(kube.URL, kube.Client()), nil - } - t.Cleanup(func() { kubeClientFactory = origFactory }) - - loads := clusterActiveRemotePodLoads("maintenance", "build") - if len(loads) != 1 || loads["good"] != 1 { - t.Fatalf("unexpected filtered loads: %#v", loads) - } -} - -func TestRunRemotePodTerminalEdgeStates(t *testing.T) { - cases := []struct { - name string - phase string - reason string - message string - logStatus int - logBody string - want string - wantErr string - }{ - {name: "success uses logs when message empty", phase: "Succeeded", logStatus: http.StatusOK, logBody: "fallback payload", want: "fallback payload"}, - {name: "success reports missing payload and log error", phase: "Succeeded", logStatus: http.StatusInternalServerError, logBody: "no logs", wantErr: "logs unavailable"}, - {name: "success reports missing payload", phase: "Succeeded", logStatus: http.StatusOK, logBody: "", wantErr: "did not return a result payload"}, - {name: "failed uses logs when message empty", phase: "Failed", logStatus: http.StatusOK, logBody: "worker failed", wantErr: "worker failed"}, - {name: "failed falls back to default reason", phase: "Failed", logStatus: http.StatusOK, logBody: "", wantErr: "remote worker failed before reporting details"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - kube := remotePodStateServer(t, tc.phase, tc.reason, tc.message, tc.logStatus, tc.logBody) - installKubeFactory(t, kube) - app := newTestApp(t) - app.settings.Namespace = "maintenance" - - got, err := app.runRemotePod("", "metis-case", map[string]any{}) - if tc.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error containing %q, got result=%q err=%v", tc.wantErr, got, err) - } - return - } - if err != nil || got != tc.want { - t.Fatalf("runRemotePod = %q err=%v", got, err) - } - }) - } -} - -func TestRunRemotePodProgressAndRequestFailures(t *testing.T) { - origFactory := kubeClientFactory - kubeClientFactory = func() (*kubeClient, error) { return nil, errors.New("offline") } - app := newTestApp(t) - if _, err := app.runRemotePod("", "metis-offline", map[string]any{}); err == nil { - t.Fatal("expected factory error") - } - kubeClientFactory = origFactory - - postFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost { - http.Error(w, "post denied", http.StatusForbidden) - return - } - w.WriteHeader(http.StatusOK) - })) - defer postFail.Close() - installKubeFactory(t, postFail) - app = newTestApp(t) - app.settings.Namespace = "maintenance" - if _, err := app.runRemotePod("", "metis-post-fail", map[string]any{}); err == nil || !strings.Contains(err.Error(), "post denied") { - t.Fatalf("expected post failure, got %v", err) - } - - stateFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodPost: - w.WriteHeader(http.StatusCreated) - case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): - http.Error(w, "state denied", http.StatusForbidden) - default: - w.WriteHeader(http.StatusOK) - } - })) - defer stateFail.Close() - installKubeFactory(t, stateFail) - app = newTestApp(t) - app.settings.Namespace = "maintenance" - if _, err := app.runRemotePod("", "metis-state-fail", map[string]any{}); err == nil || !strings.Contains(err.Error(), "state denied") { - t.Fatalf("expected state failure, got %v", err) - } - - progressLog := ProgressLogLine(RemoteProgressUpdate{Stage: "flash", ProgressPct: 92, Message: "writing", WrittenBytes: 10, TotalBytes: 20}) - progressServer := remotePodStateServer(t, "Succeeded", "Completed", `{"ok":true}`, http.StatusOK, progressLog) - installKubeFactory(t, progressServer) - app = newTestApp(t) - app.settings.Namespace = "maintenance" - job := app.newJob("replace", "titan-15", "titan-22", "/dev/sdz") - app.setJob(job.ID, func(j *Job) { - j.Status = JobRunning - j.Stage = "build" - j.StageStartedAt = time.Now() - }) - if got, err := app.runRemotePod(job.ID, "metis-progress", map[string]any{}); err != nil || got != `{"ok":true}` { - t.Fatalf("runRemotePod progress = %q err=%v", got, err) - } - if got := app.job(job.ID); got == nil || got.Written != 10 || got.Total != 20 { - t.Fatalf("expected progress update, got %#v", got) - } -} - -func TestRemotePodStateAndLogErrorBranches(t *testing.T) { - kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/state-fail"): - http.Error(w, "state unavailable", http.StatusServiceUnavailable) - case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/log"): - http.Error(w, "plain log failure", http.StatusInternalServerError) - default: - http.NotFound(w, r) - } - })) - defer kube.Close() - app := newTestApp(t) - app.settings.Namespace = "maintenance" - client := kubeClientFactoryForURL(kube.URL, kube.Client()) - - if _, err := app.remotePodState(client, "state-fail"); err == nil || !strings.Contains(err.Error(), "state unavailable") { - t.Fatalf("expected state error, got %v", err) - } - if _, err := app.remotePodLogs(client, "log-fail"); err == nil || !strings.Contains(err.Error(), "plain log failure") { - t.Fatalf("expected plain log error, got %v", err) - } - - badURL := &kubeClient{baseURL: "http://%zz", token: "tok", client: http.DefaultClient} - if _, err := app.remotePodLogs(badURL, "log-fail"); err == nil { - t.Fatal("expected log request creation error") - } - transportErr := &kubeClient{ - baseURL: "http://example.test", - token: "tok", - client: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { - return nil, errors.New("logs transport down") - })}, - } - if _, err := app.remotePodLogs(transportErr, "log-fail"); err == nil || !strings.Contains(err.Error(), "logs transport down") { - t.Fatalf("expected log transport error, got %v", err) - } -} - -func remotePodStateServer(t *testing.T, phase, reason, message string, logStatus int, logBody string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/pods"): - w.WriteHeader(http.StatusCreated) - case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/pods/"): - w.WriteHeader(http.StatusOK) - case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/") && strings.HasSuffix(r.URL.Path, "/log"): - w.WriteHeader(logStatus) - _, _ = w.Write([]byte(logBody)) - case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"): - _ = json.NewEncoder(w).Encode(map[string]any{ - "metadata": map[string]any{"name": filepath.Base(r.URL.Path)}, - "status": map[string]any{ - "phase": phase, - "reason": reason, - "message": message, - }, - }) - default: - http.NotFound(w, r) - } - })) -}