metis/pkg/service/coverage_more_test.go

package service

import (
	"encoding/json"
	"encoding/pem"
	"errors"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"metis/pkg/facts"
	"metis/pkg/inventory"
)

func TestServiceArtifactAndSnapshotPersistenceErrorBranches(t *testing.T) {
	app := newTestApp(t)

	fileParent := filepath.Join(t.TempDir(), "blocked")
	if err := os.WriteFile(fileParent, []byte("block"), 0o644); err != nil {
		t.Fatal(err)
	}
	app.settings.ArtifactStatePath = filepath.Join(fileParent, "artifacts.json")
	if err := app.persistArtifacts(); err == nil {
		t.Fatal("expected persistArtifacts to fail when parent is a file")
	}

	app.settings.SnapshotsPath = filepath.Join(fileParent, "snapshots.json")
	if err := app.persistSnapshots(); err == nil {
		t.Fatal("expected persistSnapshots to fail when parent is a file")
	}

	app.settings.TargetsPath = filepath.Join(fileParent, "targets.json")
	if err := app.persistTargets(); err == nil {
		t.Fatal("expected persistTargets to fail when parent is a file")
	}

	invalidArtifactState := filepath.Join(t.TempDir(), "artifacts.json")
	if err := os.WriteFile(invalidArtifactState, []byte("{bad-json"), 0o644); err != nil {
		t.Fatal(err)
	}
	app.settings.ArtifactStatePath = invalidArtifactState
	if err := app.loadArtifacts(); err == nil {
		t.Fatal("expected loadArtifacts to reject invalid json")
	}
}

func TestServiceReplacementAndDeviceBranches(t *testing.T) {
	app := newTestApp(t)
	ready := inventory.NodeSpec{
		Name:          "ready",
		Class:         "rpi4",
		Hostname:      "ready",
		IP:            "192.168.22.10",
		K3sRole:       "agent",
		K3sURL:        "https://192.168.22.1:6443",
		K3sToken:      "token",
		SSHUser:       "atlas",
		SSHAuthorized: []string{"ssh-ed25519 AAA"},
	}
	incomplete := inventory.NodeSpec{Name: "incomplete", Class: "rpi4"}
	class := inventory.NodeClass{Name: "rpi4", Image: "file:///tmp/base.img", Checksum: "sha256:abc"}
	app.inventory = &inventory.Inventory{Classes: []inventory.NodeClass{class}, Nodes: []inventory.NodeSpec{ready, incomplete}}
	if got := app.replacementNodes(); len(got) != 1 || got[0].Name != "ready" {
		t.Fatalf("replacementNodes = %#v", got)
	}
	if err := app.ensureReplacementReady("incomplete"); err == nil {
		t.Fatal("expected ensureReplacementReady to reject incomplete node")
	}
	if diff := diffTargets(map[string]facts.Targets{"a": {Kernel: "1"}}, map[string]facts.Targets{"a": {Kernel: "2"}, "b": {Kernel: "3"}}); len(diff) != 2 {
		t.Fatalf("diffTargets = %#v", diff)
	}

	app.recordDevices("host", []Device{{Path: "/dev/sda"}}, nil)
	if got, err := app.cachedDevices("host"); err != nil || len(got) != 1 {
		t.Fatalf("cachedDevices = %#v err=%v", got, err)
	}
	app.recordDevices("host", nil, errors.New("boom"))
	if got, err := app.cachedDevices("host"); err == nil || len(got) != 1 {
		t.Fatalf("cachedDevices error snapshot = %#v err=%v", got, err)
	}
	app.settings.DefaultFlashHost = "default-host"
	app.recordDevices("", []Device{{Path: "/dev/mmcblk0"}}, nil)
	if got, err := app.cachedDevices(""); err != nil || len(got) != 1 || got[0].Path != "/dev/mmcblk0" {
		t.Fatalf("default-host cachedDevices = %#v err=%v", got, err)
	}
	if got := deviceScore(Device{Name: "sda", Model: "SDXC Card"}); got != 50 {
		t.Fatalf("expected sd card score, got %d", got)
	}
	if _, err := app.Replace("incomplete", "titan-22", "/dev/sdz"); err == nil {
		t.Fatal("expected Replace to reject incomplete node")
	}
}

func TestServiceHarborBranches(t *testing.T) {
	harbor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/v2.0/projects"):
			_, _ = w.Write([]byte(`[]`))
		case r.Method == http.MethodPost && r.URL.Path == "/api/v2.0/projects":
			w.WriteHeader(http.StatusCreated)
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts"):
			_ = json.NewEncoder(w).Encode([]map[string]any{
				{"digest": "sha256:aaa", "push_time": "2026-04-01T10:00:00Z"},
				{"digest": "sha256:bbb", "push_time": "2026-04-01T09:00:00Z"},
			})
		case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/artifacts/"):
			w.WriteHeader(http.StatusAccepted)
		default:
			http.Error(w, "boom", http.StatusInternalServerError)
		}
	}))
	defer harbor.Close()

	app := &App{settings: Settings{
		HarborAPIBase:  harbor.URL + "/api/v2.0",
		HarborUsername: "admin",
		HarborPassword: "pw",
		HarborProject:  "metis",
		HarborRegistry: "registry.example",
	}}
	if got := app.artifactRepo("node"); got != "registry.example/metis/node" {
		t.Fatalf("artifactRepo = %q", got)
	}
	if err := app.ensureHarborProject(); err != nil {
		t.Fatalf("ensureHarborProject create: %v", err)
	}
	if err := app.pruneHarborArtifacts("node", 1); err != nil {
		t.Fatalf("pruneHarborArtifacts: %v", err)
	}
}

func TestServiceHarborErrorBranches(t *testing.T) {
	harbor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/v2.0/projects"):
			http.Error(w, "lookup failed", http.StatusInternalServerError)
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts"):
			_ = json.NewEncoder(w).Encode([]map[string]any{
				{"digest": "sha256:aaa", "push_time": "2026-04-01T10:00:00Z"},
				{"digest": "sha256:bbb", "push_time": "2026-04-01T09:00:00Z"},
			})
		case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/artifacts/"):
			http.Error(w, "delete failed", http.StatusInternalServerError)
		default:
			http.NotFound(w, r)
		}
	}))
	defer harbor.Close()

	app := &App{settings: Settings{
		HarborAPIBase:  harbor.URL + "/api/v2.0",
		HarborUsername: "admin",
		HarborPassword: "pw",
		HarborProject:  "metis",
		HarborRegistry: "registry.example",
	}}
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected ensureHarborProject error")
	}
	if err := app.pruneHarborArtifacts("node", 0); err == nil {
		t.Fatal("expected pruneHarborArtifacts error")
	}
}

func TestServiceHarborAdditionalErrorBranches(t *testing.T) {
	base := Settings{
		HarborUsername: "admin",
		HarborPassword: "pw",
		HarborProject:  "metis",
		HarborRegistry: "registry.example",
	}

	app := &App{settings: base}
	app.settings.HarborAPIBase = "://bad-url"
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected invalid Harbor lookup URL to fail")
	}
	if err := app.pruneHarborArtifacts("node", 1); err == nil {
		t.Fatal("expected invalid Harbor prune URL to fail")
	}

	app.settings.HarborAPIBase = "http://127.0.0.1:1/api/v2.0"
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected Harbor lookup connection failure")
	}
	if err := app.pruneHarborArtifacts("node", 1); err == nil {
		t.Fatal("expected Harbor artifact connection failure")
	}

	badProjectJSON := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_, _ = w.Write([]byte(`{bad-json`))
	}))
	defer badProjectJSON.Close()
	app.settings.HarborAPIBase = badProjectJSON.URL
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected malformed Harbor project response to fail")
	}

	createFailed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/projects":
			_, _ = w.Write([]byte(`[]`))
		case r.Method == http.MethodPost && r.URL.Path == "/projects":
			http.Error(w, "create failed", http.StatusInternalServerError)
		default:
			http.NotFound(w, r)
		}
	}))
	defer createFailed.Close()
	app.settings.HarborAPIBase = createFailed.URL
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected Harbor project create failure")
	}

	createConflict := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/projects":
			_, _ = w.Write([]byte(`[]`))
		case r.Method == http.MethodPost && r.URL.Path == "/projects":
			w.WriteHeader(http.StatusConflict)
		default:
			http.NotFound(w, r)
		}
	}))
	defer createConflict.Close()
	app.settings.HarborAPIBase = createConflict.URL
	if err := app.ensureHarborProject(); err != nil {
		t.Fatalf("expected Harbor project conflict to be accepted: %v", err)
	}

	postDropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/projects":
			_, _ = w.Write([]byte(`[]`))
		case r.Method == http.MethodPost && r.URL.Path == "/projects":
			conn, _, err := w.(http.Hijacker).Hijack()
			if err != nil {
				t.Fatalf("hijack Harbor POST: %v", err)
			}
			_ = conn.Close()
		default:
			http.NotFound(w, r)
		}
	}))
	defer postDropped.Close()
	app.settings.HarborAPIBase = postDropped.URL
	if err := app.ensureHarborProject(); err == nil {
		t.Fatal("expected dropped Harbor project create connection")
	}
}

func TestServiceHarborPruneAdditionalBranches(t *testing.T) {
	base := Settings{
		HarborAPIBase:  "http://unused",
		HarborUsername: "admin",
		HarborPassword: "pw",
		HarborProject:  "metis",
		HarborRegistry: "registry.example",
	}

	for name, status := range map[string]int{
		"missing": http.StatusNotFound,
		"broken":  http.StatusServiceUnavailable,
	} {
		t.Run(name, func(t *testing.T) {
			harbor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts") {
					http.Error(w, name, status)
					return
				}
				http.NotFound(w, r)
			}))
			defer harbor.Close()
			app := &App{settings: base}
			app.settings.HarborAPIBase = harbor.URL
			err := app.pruneHarborArtifacts("node", 1)
			if status == http.StatusNotFound && err != nil {
				t.Fatalf("expected missing repository to be ignored: %v", err)
			}
			if status != http.StatusNotFound && err == nil {
				t.Fatal("expected artifact list failure")
			}
		})
	}

	badArtifacts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts") {
			_, _ = w.Write([]byte(`{bad-json`))
			return
		}
		http.NotFound(w, r)
	}))
	defer badArtifacts.Close()
	app := &App{settings: base}
	app.settings.HarborAPIBase = badArtifacts.URL
	if err := app.pruneHarborArtifacts("node", 1); err == nil {
		t.Fatal("expected malformed artifact list to fail")
	}

	deletes := 0
	pruneOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts"):
			_ = json.NewEncoder(w).Encode([]map[string]any{
				{"digest": "sha256:oldest", "push_time": "2026-04-01T08:00:00Z"},
				{"digest": "sha256:newest", "push_time": "2026-04-01T10:00:00Z"},
			})
		case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/artifacts/"):
			deletes++
			w.WriteHeader(http.StatusOK)
		default:
			http.NotFound(w, r)
		}
	}))
	defer pruneOK.Close()
	app.settings.HarborAPIBase = pruneOK.URL
	if err := app.pruneHarborArtifacts("node", 1); err != nil {
		t.Fatalf("expected old artifact prune to succeed: %v", err)
	}
	if deletes != 1 {
		t.Fatalf("expected one old artifact delete, got %d", deletes)
	}

	pruneDeleteFailed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts"):
			_ = json.NewEncoder(w).Encode([]map[string]any{
				{"digest": "sha256:stale", "push_time": "2026-04-01T08:00:00Z"},
			})
		case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/artifacts/"):
			http.Error(w, "delete failed", http.StatusInternalServerError)
		default:
			http.NotFound(w, r)
		}
	}))
	defer pruneDeleteFailed.Close()
	app.settings.HarborAPIBase = pruneDeleteFailed.URL
	if err := app.pruneHarborArtifacts("node", 0); err == nil {
		t.Fatal("expected artifact delete failure")
	}

	deleteDropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/artifacts"):
			_ = json.NewEncoder(w).Encode([]map[string]any{
				{"digest": "sha256:stale", "push_time": "2026-04-01T08:00:00Z"},
			})
		case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/artifacts/"):
			conn, _, err := w.(http.Hijacker).Hijack()
			if err != nil {
				t.Fatalf("hijack Harbor DELETE: %v", err)
			}
			_ = conn.Close()
		default:
			http.NotFound(w, r)
		}
	}))
	defer deleteDropped.Close()
	app.settings.HarborAPIBase = deleteDropped.URL
	if err := app.pruneHarborArtifacts("node", 0); err == nil {
		t.Fatal("expected dropped Harbor artifact delete connection")
	}
}

func TestServiceClusterAndRemotePodBranches(t *testing.T) {
	origTokenPath := kubeServiceAccountTokenPath
	origCAPath := kubeServiceAccountCAPath
	dir := t.TempDir()
	kubeServiceAccountTokenPath = filepath.Join(dir, "token")
	kubeServiceAccountCAPath = filepath.Join(dir, "ca.crt")
	t.Cleanup(func() {
		kubeServiceAccountTokenPath = origTokenPath
		kubeServiceAccountCAPath = origCAPath
	})
	if err := os.WriteFile(kubeServiceAccountTokenPath, []byte("tok"), 0o644); err != nil {
		t.Fatal(err)
	}
	t.Setenv("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
	t.Setenv("KUBERNETES_SERVICE_PORT", "443")

	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
			_ = json.NewEncoder(w).Encode(map[string]any{
				"items": []any{
					map[string]any{
						"metadata": map[string]any{"name": "b", "labels": map[string]string{"kubernetes.io/arch": "arm64", "node-role.kubernetes.io/worker": "true"}},
						"spec":     map[string]any{"unschedulable": false},
					},
					map[string]any{
						"metadata": map[string]any{"name": "a", "labels": map[string]string{"kubernetes.io/arch": "arm64", "node-role.kubernetes.io/worker": "true"}},
						"spec":     map[string]any{"unschedulable": false},
					},
				},
			})
		case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/pods"):
			w.WriteHeader(http.StatusCreated)
		case r.Method == http.MethodDelete:
			w.WriteHeader(http.StatusOK)
		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/log"):
			_, _ = w.Write([]byte("pod logs"))
		case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"):
			_ = json.NewEncoder(w).Encode(map[string]any{
				"metadata": map[string]any{"name": filepath.Base(r.URL.Path)},
				"status": map[string]any{
					"phase":   "Succeeded",
					"message": `{"dest_path":"/tmp/out.img","verified":true,"verification_kind":"image-file","verification_summary":"Verified image layout at /tmp/out.img; boot and writable partitions are present."}`,
					"reason":  "Completed",
				},
			})
		default:
			http.NotFound(w, r)
		}
	}))
	defer srv.Close()
	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
	if err := os.WriteFile(kubeServiceAccountCAPath, certPEM, 0o644); err != nil {
		t.Fatal(err)
	}

	client, err := inClusterKubeClient()
	if err != nil {
		t.Fatalf("inClusterKubeClient: %v", err)
	}
	client.baseURL = srv.URL
	client.client = srv.Client()
	kubeClientFactory = func() (*kubeClient, error) { return client, nil }
	t.Cleanup(func() { kubeClientFactory = inClusterKubeClient })

	var nodePayload map[string]any
	if err := client.jsonRequest(http.MethodGet, "/api/v1/nodes", nil, &nodePayload); err != nil {
		t.Fatalf("jsonRequest: %v", err)
	}
	if err := client.deleteRequest("/api/v1/nodes/a"); err != nil {
		t.Fatalf("deleteRequest: %v", err)
	}
	if nodes := clusterNodes(); len(nodes) != 2 || nodes[0].Name != "a" {
		t.Fatalf("clusterNodes = %#v", nodes)
	}

	app := newTestApp(t)
	app.settings.Namespace = "maintenance"
	app.settings.RunnerImageARM64 = "runner:arm64"
	state, err := app.remotePodState(client, "metis-build-test")
	if err != nil {
		t.Fatalf("remotePodState: %v", err)
	}
	if state.Phase != "Succeeded" || state.Message == "" {
		t.Fatalf("remotePodState = %#v", state)
	}
	logs, err := app.remotePodLogs(client, "metis-build-test")
	if err != nil || logs != "pod logs" {
		t.Fatalf("remotePodLogs = %q err=%v", logs, err)
	}
	if got := app.podImageForArch("amd64"); got != "" {
		t.Fatalf("podImageForArch fallback = %q", got)
	}
	if got := app.podImageForArch("arm64"); got != "runner:arm64" {
		t.Fatalf("podImageForArch arm64 = %q", got)
	}

	job := app.newJob("build", "titan-15", "titan-22", "/dev/sdz")
	app.settings.HarborAPIBase = ""
	app.runBuild(job, false)
	if got := app.job(job.ID); got == nil || got.Status != JobError {
		t.Fatalf("runBuild should fail without harbor creds: %#v", got)
	}
	if _, err := app.Replace("incomplete", "titan-22", "/dev/sdz"); err == nil {
		t.Fatal("expected Replace to reject incomplete node")
	}
}