2026-04-22 03:57:55 -03:00
|
|
|
package service
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"encoding/json"
|
2026-04-22 04:07:13 -03:00
|
|
|
"errors"
|
2026-04-22 03:57:55 -03:00
|
|
|
"net/http"
|
|
|
|
|
"net/http/httptest"
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"metis/pkg/facts"
|
|
|
|
|
"metis/pkg/sentinel"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestScratchHealthAnnotations(t *testing.T) {
|
2026-04-22 04:07:13 -03:00
|
|
|
if err := newTestApp(t).syncScratchAnnotations(SnapshotRecord{}); err != nil {
|
|
|
|
|
t.Fatalf("nil scratch sync should be a noop: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if status, detail := scratchStatusDetail(nil); status != "missing" || detail != "no-scratch-snapshot" {
|
|
|
|
|
t.Fatalf("nil scratch detail mismatch: %s %s", status, detail)
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-22 03:57:55 -03:00
|
|
|
observed := time.Date(2026, 4, 22, 6, 45, 0, 0, time.UTC)
|
|
|
|
|
annotations := scratchHealthAnnotations(&facts.USBScratch{
|
|
|
|
|
Mountpoint: "/mnt/astraios",
|
|
|
|
|
UUID: "usb-1",
|
|
|
|
|
FS: "ext4",
|
|
|
|
|
MountHealthy: true,
|
|
|
|
|
UUIDHealthy: true,
|
|
|
|
|
BindHealthy: true,
|
|
|
|
|
BindTargets: []facts.USBBindTarget{{Path: "/var/log/pods", Healthy: true}, {Path: "/var/tmp", Healthy: true}},
|
|
|
|
|
}, observed)
|
|
|
|
|
if annotations["maintenance.bstein.dev/usb-scratch-status"] != "ok" || annotations["maintenance.bstein.dev/astraios-detail"] != "healthy" {
|
|
|
|
|
t.Fatalf("unexpected healthy annotations: %#v", annotations)
|
|
|
|
|
}
|
|
|
|
|
if annotations["maintenance.bstein.dev/usb-scratch-selector"] != "UUID=usb-1" {
|
|
|
|
|
t.Fatalf("selector annotation missing: %#v", annotations)
|
|
|
|
|
}
|
|
|
|
|
if annotations["maintenance.bstein.dev/astraios-managed-paths"] != "/var/log/pods_/var/tmp" {
|
|
|
|
|
t.Fatalf("managed paths annotation mismatch: %#v", annotations)
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-22 04:07:13 -03:00
|
|
|
labelAnnotations := scratchHealthAnnotations(&facts.USBScratch{
|
|
|
|
|
Mountpoint: "/mnt/scratch",
|
|
|
|
|
Label: "scratch-a",
|
|
|
|
|
MountHealthy: true,
|
|
|
|
|
LabelHealthy: true,
|
|
|
|
|
BindHealthy: true,
|
|
|
|
|
}, observed)
|
|
|
|
|
if labelAnnotations["maintenance.bstein.dev/usb-scratch-selector"] != "LABEL=scratch-a" {
|
|
|
|
|
t.Fatalf("label selector annotation mismatch: %#v", labelAnnotations)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
status, detail := scratchStatusDetail(&facts.USBScratch{Label: "scratch-a", MountHealthy: false, LabelHealthy: false, BindHealthy: false})
|
|
|
|
|
if status != "error" || !strings.Contains(detail, "mount-unhealthy") || !strings.Contains(detail, "label-mismatch") || !strings.Contains(detail, "bind-mount-incomplete") {
|
2026-04-22 03:57:55 -03:00
|
|
|
t.Fatalf("unexpected unhealthy detail: %s %s", status, detail)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestStoreSnapshotPatchesNodeAnnotations(t *testing.T) {
|
|
|
|
|
var patchPath string
|
|
|
|
|
var patchContentType string
|
|
|
|
|
var patchBody map[string]any
|
|
|
|
|
kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
if r.Method != http.MethodPatch || r.URL.Path != "/api/v1/nodes/titan-04" {
|
|
|
|
|
http.NotFound(w, r)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
patchPath = r.URL.Path
|
|
|
|
|
patchContentType = r.Header.Get("Content-Type")
|
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil {
|
|
|
|
|
t.Fatalf("decode patch body: %v", err)
|
|
|
|
|
}
|
|
|
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"status": "ok"})
|
|
|
|
|
}))
|
|
|
|
|
defer kube.Close()
|
|
|
|
|
|
|
|
|
|
origFactory := kubeClientFactory
|
|
|
|
|
kubeClientFactory = func() (*kubeClient, error) {
|
|
|
|
|
return kubeClientFactoryForURL(kube.URL, kube.Client()), nil
|
|
|
|
|
}
|
|
|
|
|
t.Cleanup(func() { kubeClientFactory = origFactory })
|
|
|
|
|
|
|
|
|
|
app := newTestApp(t)
|
|
|
|
|
if err := app.StoreSnapshot(SnapshotRecord{
|
|
|
|
|
Node: "titan-04",
|
|
|
|
|
CollectedAt: time.Date(2026, 4, 22, 6, 50, 0, 0, time.UTC),
|
|
|
|
|
Snapshot: sentinel.Snapshot{
|
|
|
|
|
Hostname: "titan-04",
|
|
|
|
|
USBScratch: &facts.USBScratch{
|
|
|
|
|
Mountpoint: "/mnt/astraios",
|
|
|
|
|
UUID: "usb-1",
|
|
|
|
|
MountHealthy: true,
|
|
|
|
|
UUIDHealthy: true,
|
|
|
|
|
BindHealthy: true,
|
|
|
|
|
BindTargets: []facts.USBBindTarget{{Path: "/var/log/pods", Healthy: true}},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatalf("StoreSnapshot: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if patchPath != "/api/v1/nodes/titan-04" || patchContentType != "application/merge-patch+json" {
|
|
|
|
|
t.Fatalf("patch request mismatch: path=%q content-type=%q", patchPath, patchContentType)
|
|
|
|
|
}
|
|
|
|
|
metadata := patchBody["metadata"].(map[string]any)
|
|
|
|
|
annotations := metadata["annotations"].(map[string]any)
|
|
|
|
|
if annotations["maintenance.bstein.dev/usb-scratch-status"] != "ok" || annotations["maintenance.bstein.dev/astraios-selector"] != "UUID=usb-1" {
|
|
|
|
|
t.Fatalf("annotation patch mismatch: %#v", annotations)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-04-22 04:07:13 -03:00
|
|
|
|
|
|
|
|
func TestStoreSnapshotRecordsAnnotationPatchFailure(t *testing.T) {
|
|
|
|
|
kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
|
|
|
http.Error(w, "denied", http.StatusForbidden)
|
|
|
|
|
}))
|
|
|
|
|
defer kube.Close()
|
|
|
|
|
|
|
|
|
|
origFactory := kubeClientFactory
|
|
|
|
|
kubeClientFactory = func() (*kubeClient, error) {
|
|
|
|
|
return kubeClientFactoryForURL(kube.URL, kube.Client()), nil
|
|
|
|
|
}
|
|
|
|
|
t.Cleanup(func() { kubeClientFactory = origFactory })
|
|
|
|
|
|
|
|
|
|
app := newTestApp(t)
|
|
|
|
|
if err := app.StoreSnapshot(SnapshotRecord{
|
|
|
|
|
Node: "titan-04",
|
|
|
|
|
CollectedAt: time.Date(2026, 4, 22, 6, 55, 0, 0, time.UTC),
|
|
|
|
|
Snapshot: sentinel.Snapshot{
|
|
|
|
|
Hostname: "titan-04",
|
|
|
|
|
USBScratch: &facts.USBScratch{
|
|
|
|
|
Mountpoint: "/mnt/astraios",
|
|
|
|
|
MountHealthy: true,
|
|
|
|
|
BindHealthy: true,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatalf("StoreSnapshot should keep the snapshot even if annotation sync fails: %v", err)
|
|
|
|
|
}
|
|
|
|
|
state := app.State("")
|
|
|
|
|
found := false
|
|
|
|
|
for _, event := range state.Events {
|
|
|
|
|
if event.Kind == "sentinel.annotation" {
|
|
|
|
|
found = true
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if !found {
|
|
|
|
|
t.Fatalf("expected annotation warning event, got %#v", state.Events)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestPatchNodeAnnotationsNoopsAndClientErrors(t *testing.T) {
|
|
|
|
|
if err := patchNodeAnnotations("", map[string]string{"a": "b"}); err != nil {
|
|
|
|
|
t.Fatalf("empty node should be a noop: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if err := patchNodeAnnotations("node", nil); err != nil {
|
|
|
|
|
t.Fatalf("empty annotations should be a noop: %v", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
origFactory := kubeClientFactory
|
|
|
|
|
kubeClientFactory = func() (*kubeClient, error) { return nil, errors.New("offline") }
|
|
|
|
|
if err := patchNodeAnnotations("node", map[string]string{"a": "b"}); err == nil || !strings.Contains(err.Error(), "offline") {
|
|
|
|
|
t.Fatalf("expected factory error, got %v", err)
|
|
|
|
|
}
|
|
|
|
|
t.Cleanup(func() { kubeClientFactory = origFactory })
|
|
|
|
|
|
|
|
|
|
client := kubeClientFactoryForURL("http://127.0.0.1", &http.Client{Transport: failingRoundTripper{}})
|
|
|
|
|
if err := client.mergePatch("/api/v1/nodes/node", map[string]string{"a": "b"}); err == nil || !strings.Contains(err.Error(), "transport down") {
|
|
|
|
|
t.Fatalf("expected transport error, got %v", err)
|
|
|
|
|
}
|
|
|
|
|
if err := client.mergePatch("/api/v1/nodes/node", map[string]any{"bad": func() {}}); err == nil {
|
|
|
|
|
t.Fatal("expected marshal error")
|
|
|
|
|
}
|
|
|
|
|
badURLClient := kubeClientFactoryForURL("http://[::1", http.DefaultClient)
|
|
|
|
|
if err := badURLClient.mergePatch("/api/v1/nodes/node", map[string]string{"a": "b"}); err == nil {
|
|
|
|
|
t.Fatal("expected request construction error")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type failingRoundTripper struct{}
|
|
|
|
|
|
|
|
|
|
func (failingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
|
|
|
|
|
return nil, errors.New("transport down")
|
|
|
|
|
}
|