diff --git a/cmd/metis/gate_test.go b/cmd/metis/gate_test.go index b75c01e..b213380 100644 --- a/cmd/metis/gate_test.go +++ b/cmd/metis/gate_test.go @@ -63,6 +63,14 @@ exit 0`, ;; esac exit 0`, + "losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then + printf '/dev/loop9\n' + exit 0 +fi +exit 0`, + "mount": `mkdir -p "${2:-}" +exit 0`, + "umount": `exit 0`, }) t.Setenv("PATH", fakeTools+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("METIS_INVENTORY_PATH", invPath) diff --git a/cmd/metis/main_test.go b/cmd/metis/main_test.go index 418ba53..f1d343f 100644 --- a/cmd/metis/main_test.go +++ b/cmd/metis/main_test.go @@ -86,6 +86,14 @@ if [[ "${1:-}" == "-R" ]]; then esac fi exit 0`, + "losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then + printf '/dev/loop9\n' + exit 0 +fi +exit 0`, + "mount": `mkdir -p "${2:-}" +exit 0`, + "umount": `exit 0`, }) t.Setenv("PATH", rootTools+string(os.PathListSeparator)+os.Getenv("PATH")) stdout, _ = captureStreams(t, func() { @@ -182,6 +190,14 @@ exit 0`, ;; esac exit 0`, + "losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then + printf '/dev/loop9\n' + exit 0 +fi +exit 0`, + "mount": `mkdir -p "${2:-}" +exit 0`, + "umount": `exit 0`, }) t.Setenv("PATH", fakeTools+string(os.PathListSeparator)+os.Getenv("PATH")) diff --git a/cmd/metis/remote_cmd.go b/cmd/metis/remote_cmd.go index 372ba57..d2b6c32 100644 --- a/cmd/metis/remote_cmd.go +++ b/cmd/metis/remote_cmd.go @@ -108,6 +108,10 @@ func remoteBuildCmd(args []string) { fatalf("inject rootfs: %v", err) } emitStageProgress("build", 58, fmt.Sprintf("Built the replacement image filesystem for %s", *node)) + emitStageProgress("build", 59, fmt.Sprintf("Injecting boot-partition recovery files for %s", *node)) + if err := image.InjectBootFiles(output, files); err != nil { + fatalf("inject boot files: %v", err) + } emitStageProgress("build", 60, fmt.Sprintf("Compressing the replacement image for %s before upload", *node)) if err := exec.Command("xz", "-T0", "-z", "-f", output).Run(); err != nil { fatalf("xz compress: %v", err) diff --git a/cmd/metis/remote_edge_test.go b/cmd/metis/remote_edge_test.go index 5207a12..6052c47 100644 --- a/cmd/metis/remote_edge_test.go +++ b/cmd/metis/remote_edge_test.go @@ -415,8 +415,17 @@ if [[ "${1:-}" == "-R" ]]; then esac fi exit 0`, - "xz": xzBody, - "oras": orasBody, + "xz": xzBody, + "oras": orasBody, + "losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then + printf '/dev/loop9 +' + exit 0 +fi +exit 0`, + "mount": `mkdir -p "${2:-}" +exit 0`, + "umount": `exit 0`, "sync": `exit 0`, "blockdev": `exit 0`, "cp-base": `cp "` + baseImage + `" "$1"`, diff --git a/pkg/image/rootfs.go b/pkg/image/rootfs.go index c713a89..6ee3cd0 100644 --- a/pkg/image/rootfs.go +++ b/pkg/image/rootfs.go @@ -12,6 +12,7 @@ import ( "strings" "metis/pkg/inject" + "metis/pkg/mount" ) type partitionTable struct { @@ -53,6 +54,42 @@ func InjectRootFS(imagePath string, files []inject.FileSpec) error { return InjectRootFSWithProgress(imagePath, files, nil) } +// InjectBootFiles loop-mounts a raw image long enough to materialize any +// boot-partition files such as NoCloud user-data and meta-data. +func InjectBootFiles(imagePath string, files []inject.FileSpec) error { + bootFiles := make([]inject.FileSpec, 0, len(files)) + for _, f := range files { + if !f.RootFS { + bootFiles = append(bootFiles, f) + } + } + if len(bootFiles) == 0 { + return nil + } + + mounted, err := mount.Setup(imagePath) + if err != nil { + return fmt.Errorf("mount image for boot injection: %w", err) + } + defer mount.Teardown(mounted) + + inj := inject.Injector{BootPath: mounted.BootPath} + if err := inj.Write(bootFiles); err != nil { + return fmt.Errorf("write boot files: %w", err) + } + return nil +} + +// InjectImage materializes both rootfs and boot-partition files into a raw +// image. Rootfs content is rewritten without a mount; boot files are applied via +// a short loop-mount because the Raspberry Pi boot partition is FAT. +func InjectImage(imagePath string, files []inject.FileSpec) error { + if err := InjectRootFS(imagePath, files); err != nil { + return err + } + return InjectBootFiles(imagePath, files) +} + // InjectRootFSWithProgress emits coarse step changes while rewriting the root partition. func InjectRootFSWithProgress(imagePath string, files []inject.FileSpec, progress RootFSProgressFunc) error { rootFiles := make([]inject.FileSpec, 0, len(files)) diff --git a/pkg/image/rootfs_test.go b/pkg/image/rootfs_test.go index 7974594..1125e52 100644 --- a/pkg/image/rootfs_test.go +++ b/pkg/image/rootfs_test.go @@ -313,6 +313,133 @@ exit 0`}) }) } +func TestInjectBootFilesWithFakes(t *testing.T) { + dumpDir := filepath.Join(t.TempDir(), "dump") + t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir) + scripts := fakeImageMountCommands(t, nil) + t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH")) + + imagePath := filepath.Join(t.TempDir(), "image.img") + if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil { + t.Fatal(err) + } + files := []inject.FileSpec{{Path: "user-data", Content: []byte("#cloud-config\n"), Mode: 0o644, RootFS: false}} + if err := InjectBootFiles(imagePath, files); err != nil { + t.Fatalf("InjectBootFiles: %v", err) + } + if got := readDumpedFile(t, dumpDir, "user-data"); got != "#cloud-config\n" { + t.Fatalf("dumped user-data = %q", got) + } +} + +func TestInjectBootFilesNoopsWithoutBootFiles(t *testing.T) { + imagePath := filepath.Join(t.TempDir(), "image.img") + if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil { + t.Fatal(err) + } + if err := InjectBootFiles(imagePath, []inject.FileSpec{{Path: "etc/metis/node.json", Content: []byte("{}"), Mode: 0o600, RootFS: true}}); err != nil { + t.Fatalf("InjectBootFiles root-only: %v", err) + } +} + +func TestInjectBootFilesReportsMountFailures(t *testing.T) { + scripts := fakeImageMountCommands(t, map[string]string{"mount": `exit 9`}) + t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH")) + + imagePath := filepath.Join(t.TempDir(), "image.img") + if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil { + t.Fatal(err) + } + err := InjectBootFiles(imagePath, []inject.FileSpec{{Path: "user-data", Content: []byte("boot"), Mode: 0o644, RootFS: false}}) + if err == nil { + t.Fatal("expected mount failure") + } +} + +func TestInjectImageWithFakes(t *testing.T) { + rootScripts := fakeRootfsCommands(t, true) + mountScripts := fakeImageMountCommands(t, nil) + t.Setenv("PATH", rootScripts+string(os.PathListSeparator)+mountScripts+string(os.PathListSeparator)+os.Getenv("PATH")) + + dumpDir := filepath.Join(t.TempDir(), "dump") + t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir) + imagePath := filepath.Join(t.TempDir(), "image.img") + if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil { + t.Fatal(err) + } + files := []inject.FileSpec{ + {Path: "etc/metis/firstboot.env", Content: []byte(`METIS_HOSTNAME='titan-13' +`), Mode: 0o600, RootFS: true}, + {Path: "user-data", Content: []byte(`#cloud-config +`), Mode: 0o644, RootFS: false}, + } + if err := InjectImage(imagePath, files); err != nil { + t.Fatalf("InjectImage: %v", err) + } + if got := readDumpedFile(t, dumpDir, "user-data"); got != `#cloud-config +` { + t.Fatalf("dumped user-data = %q", got) + } +} + +func readDumpedFile(t *testing.T, dumpDir, fileName string) string { + t.Helper() + var content string + err := filepath.Walk(dumpDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || filepath.Base(path) != fileName { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + content = string(data) + return filepath.SkipAll + }) + if err != nil { + t.Fatalf("walk dump dir: %v", err) + } + if content == "" { + t.Fatalf("did not find %s under %s", fileName, dumpDir) + } + return content +} + +func fakeImageMountCommands(t *testing.T, overrides map[string]string) string { + t.Helper() + dir := t.TempDir() + scripts := map[string]string{ + "losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then + printf '/dev/loop9\n' + exit 0 +fi +exit 0`, + "mount": `mkdir -p "${2:-}" +exit 0`, + "umount": `target="${1:-}" +dump="${METIS_FAKE_UMOUNT_DUMP_DIR:-}" +if [[ -n "$dump" && -d "$target" ]]; then + base="$(basename "$target")" + mkdir -p "$dump/$base" + cp -a "$target"/. "$dump/$base/" 2>/dev/null || true +fi +exit 0`, + } + for name, body := range overrides { + scripts[name] = body + } + for name, body := range scripts { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -eu\n"+body+"\n"), 0o755); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + return dir +} + func fakeRootfsCommands(t *testing.T, includeLinux bool) string { t.Helper() dir := t.TempDir() diff --git a/pkg/plan/image_build.go b/pkg/plan/image_build.go index d06c7fa..d2d2b9f 100644 --- a/pkg/plan/image_build.go +++ b/pkg/plan/image_build.go @@ -35,8 +35,8 @@ func BuildImageFile(ctx context.Context, inv *inventory.Inventory, nodeName, cac if err != nil { return fmt.Errorf("resolve files: %w", err) } - if err := image.InjectRootFS(output, files); err != nil { - return fmt.Errorf("inject rootfs: %w", err) + if err := image.InjectImage(output, files); err != nil { + return fmt.Errorf("inject image: %w", err) } return nil } diff --git a/pkg/plan/workflow_test.go b/pkg/plan/workflow_test.go index 44667b2..ad06389 100644 --- a/pkg/plan/workflow_test.go +++ b/pkg/plan/workflow_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "os" "path/filepath" + "strings" "testing" "metis/pkg/inventory" @@ -78,7 +79,8 @@ func TestExecuteAndBuildImageFileWithFakes(t *testing.T) { func TestBuildImageFileMaterializesRootFS(t *testing.T) { rootTools := fakeRootfsTools(t) - t.Setenv("PATH", rootTools+string(os.PathListSeparator)+os.Getenv("PATH")) + mountTools := fakeMountTools(t) + t.Setenv("PATH", rootTools+string(os.PathListSeparator)+mountTools+string(os.PathListSeparator)+os.Getenv("PATH")) dir := t.TempDir() rawImage := filepath.Join(dir, "base.img") @@ -104,6 +106,8 @@ func TestBuildImageFileMaterializesRootFS(t *testing.T) { SSHAuthorized: []string{"ssh-ed25519 AAA"}, }}, } + dumpDir := filepath.Join(dir, "dump") + t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir) out := filepath.Join(dir, "output.img") if err := BuildImageFile(context.Background(), inv, "n1", filepath.Join(dir, "cache"), out); err != nil { t.Fatalf("BuildImageFile: %v", err) @@ -111,6 +115,10 @@ func TestBuildImageFileMaterializesRootFS(t *testing.T) { if _, err := os.Stat(out); err != nil { t.Fatalf("expected output image: %v", err) } + userData := findDumpedFileContent(t, dumpDir, "user-data") + if !strings.Contains(userData, "metis-apply-node-identity.sh") { + t.Fatalf("expected boot cloud-init hook in user-data, got: %s", userData) + } } func TestMaybeInjectNoopsWhenEnvUnset(t *testing.T) { @@ -183,11 +191,45 @@ func fakeMountTools(t *testing.T) string { exit 0 fi exit 0`) - write("mount", `exit 0`) - write("umount", `exit 0`) + write("mount", `mkdir -p "${2:-}" +exit 0`) + write("umount", `target="${1:-}" +dump="${METIS_FAKE_UMOUNT_DUMP_DIR:-}" +if [[ -n "$dump" && -d "$target" ]]; then + base="$(basename "$target")" + mkdir -p "$dump/$base" + cp -a "$target"/. "$dump/$base/" 2>/dev/null || true +fi +exit 0`) return dir } +func findDumpedFileContent(t *testing.T, dumpDir, fileName string) string { + t.Helper() + var content string + err := filepath.Walk(dumpDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || filepath.Base(path) != fileName { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + content = string(data) + return filepath.SkipAll + }) + if err != nil { + t.Fatalf("walk dumped files: %v", err) + } + if content == "" { + t.Fatalf("did not find dumped %s under %s", fileName, dumpDir) + } + return content +} + func imageChecksum(t *testing.T, path string) string { t.Helper() data, err := os.ReadFile(path) diff --git a/pkg/service/remote_helpers.go b/pkg/service/remote_helpers.go index c77e7c2..e3e96be 100644 --- a/pkg/service/remote_helpers.go +++ b/pkg/service/remote_helpers.go @@ -284,19 +284,25 @@ func (a *App) remoteBuildPodSpec(name, host, image, node, nodeHostname, artifact "--harbor-registry", a.settings.HarborRegistry, ), }, - "securityContext": map[string]any{"runAsUser": 0, "runAsGroup": 0}, + "securityContext": map[string]any{"privileged": true, "runAsUser": 0, "runAsGroup": 0}, "env": desiredEnv, "envFrom": []map[string]any{ {"configMapRef": map[string]any{"name": "metis"}}, }, "volumeMounts": []map[string]any{ {"name": "workspace", "mountPath": "/workspace"}, + {"name": "host-dev", "mountPath": "/dev"}, + {"name": "host-sys", "mountPath": "/sys", "readOnly": true}, + {"name": "host-udev", "mountPath": "/run/udev", "readOnly": true}, }, }, }, "imagePullSecrets": []map[string]string{{"name": "harbor-regcred"}}, "volumes": []map[string]any{ {"name": "workspace", "hostPath": map[string]any{"path": workspaceHostPath, "type": "DirectoryOrCreate"}}, + {"name": "host-dev", "hostPath": map[string]any{"path": "/dev"}}, + {"name": "host-sys", "hostPath": map[string]any{"path": "/sys"}}, + {"name": "host-udev", "hostPath": map[string]any{"path": "/run/udev"}}, }, }, } diff --git a/pkg/service/remote_helpers_test.go b/pkg/service/remote_helpers_test.go index 17acf83..cf42282 100644 --- a/pkg/service/remote_helpers_test.go +++ b/pkg/service/remote_helpers_test.go @@ -277,12 +277,18 @@ func TestRemoteWorkspaceAndHostTmpPathsPreferUsbScratchRoots(t *testing.T) { t.Fatalf("expected node password exports in vault template: %#v", metadataAnnotations) } buildSecurity := buildContainer["securityContext"].(map[string]any) + if got := buildSecurity["privileged"]; got != true { + t.Fatalf("build privileged = %v", got) + } if got := buildSecurity["runAsUser"]; got != 0 { t.Fatalf("build runAsUser = %v", got) } if got := buildSecurity["runAsGroup"]; got != 0 { t.Fatalf("build runAsGroup = %v", got) } + if got := buildVolumes[1]["hostPath"].(map[string]any)["path"]; got != "/dev" { + t.Fatalf("build host-dev hostPath = %v", got) + } flashSpec := app.remoteFlashPodSpec("metis-flash-123", "titan-04", "runner:arm64", "titan-10", hostTmpDevicePath, "registry.example/metis/titan-10") flashVolumes := flashSpec["spec"].(map[string]any)["volumes"].([]map[string]any)