recovery(metis): inject boot recovery files in remote builds

cmd/metis/gate_test.go

@@ -63,6 +63,14 @@ exit 0`,
     ;;
 esac
 exit 0`,
+		"losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then
+  printf '/dev/loop9\n'
+  exit 0
+fi
+exit 0`,
+		"mount": `mkdir -p "${2:-}"
+exit 0`,
+		"umount": `exit 0`,
 	})
 	t.Setenv("PATH", fakeTools+string(os.PathListSeparator)+os.Getenv("PATH"))
 	t.Setenv("METIS_INVENTORY_PATH", invPath)

cmd/metis/main_test.go

@@ -86,6 +86,14 @@ if [[ "${1:-}" == "-R" ]]; then
   esac
 fi
 exit 0`,
+		"losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then
+  printf '/dev/loop9\n'
+  exit 0
+fi
+exit 0`,
+		"mount": `mkdir -p "${2:-}"
+exit 0`,
+		"umount": `exit 0`,
 	})
 	t.Setenv("PATH", rootTools+string(os.PathListSeparator)+os.Getenv("PATH"))
 	stdout, _ = captureStreams(t, func() {
@@ -182,6 +190,14 @@ exit 0`,
     ;;
 esac
 exit 0`,
+		"losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then
+  printf '/dev/loop9\n'
+  exit 0
+fi
+exit 0`,
+		"mount": `mkdir -p "${2:-}"
+exit 0`,
+		"umount": `exit 0`,
 	})
 	t.Setenv("PATH", fakeTools+string(os.PathListSeparator)+os.Getenv("PATH"))
 

cmd/metis/remote_cmd.go

@@ -108,6 +108,10 @@ func remoteBuildCmd(args []string) {
 		fatalf("inject rootfs: %v", err)
 	}
 	emitStageProgress("build", 58, fmt.Sprintf("Built the replacement image filesystem for %s", *node))
+	emitStageProgress("build", 59, fmt.Sprintf("Injecting boot-partition recovery files for %s", *node))
+	if err := image.InjectBootFiles(output, files); err != nil {
+		fatalf("inject boot files: %v", err)
+	}
 	emitStageProgress("build", 60, fmt.Sprintf("Compressing the replacement image for %s before upload", *node))
 	if err := exec.Command("xz", "-T0", "-z", "-f", output).Run(); err != nil {
 		fatalf("xz compress: %v", err)

cmd/metis/remote_edge_test.go

@@ -415,8 +415,17 @@ if [[ "${1:-}" == "-R" ]]; then
   esac
 fi
 exit 0`,
-		"xz":       xzBody,
+		"xz":   xzBody,
-		"oras":     orasBody,
+		"oras": orasBody,
+		"losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then
+  printf '/dev/loop9
+'
+  exit 0
+fi
+exit 0`,
+		"mount": `mkdir -p "${2:-}"
+exit 0`,
+		"umount":   `exit 0`,
 		"sync":     `exit 0`,
 		"blockdev": `exit 0`,
 		"cp-base":  `cp "` + baseImage + `" "$1"`,

pkg/image/rootfs.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"metis/pkg/inject"
+	"metis/pkg/mount"
 )
 
 type partitionTable struct {
@@ -53,6 +54,42 @@ func InjectRootFS(imagePath string, files []inject.FileSpec) error {
 	return InjectRootFSWithProgress(imagePath, files, nil)
 }
 
+// InjectBootFiles loop-mounts a raw image long enough to materialize any
+// boot-partition files such as NoCloud user-data and meta-data.
+func InjectBootFiles(imagePath string, files []inject.FileSpec) error {
+	bootFiles := make([]inject.FileSpec, 0, len(files))
+	for _, f := range files {
+		if !f.RootFS {
+			bootFiles = append(bootFiles, f)
+		}
+	}
+	if len(bootFiles) == 0 {
+		return nil
+	}
+
+	mounted, err := mount.Setup(imagePath)
+	if err != nil {
+		return fmt.Errorf("mount image for boot injection: %w", err)
+	}
+	defer mount.Teardown(mounted)
+
+	inj := inject.Injector{BootPath: mounted.BootPath}
+	if err := inj.Write(bootFiles); err != nil {
+		return fmt.Errorf("write boot files: %w", err)
+	}
+	return nil
+}
+
+// InjectImage materializes both rootfs and boot-partition files into a raw
+// image. Rootfs content is rewritten without a mount; boot files are applied via
+// a short loop-mount because the Raspberry Pi boot partition is FAT.
+func InjectImage(imagePath string, files []inject.FileSpec) error {
+	if err := InjectRootFS(imagePath, files); err != nil {
+		return err
+	}
+	return InjectBootFiles(imagePath, files)
+}
+
 // InjectRootFSWithProgress emits coarse step changes while rewriting the root partition.
 func InjectRootFSWithProgress(imagePath string, files []inject.FileSpec, progress RootFSProgressFunc) error {
 	rootFiles := make([]inject.FileSpec, 0, len(files))

pkg/image/rootfs_test.go

@@ -313,6 +313,133 @@ exit 0`})
 	})
 }
 
+func TestInjectBootFilesWithFakes(t *testing.T) {
+	dumpDir := filepath.Join(t.TempDir(), "dump")
+	t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir)
+	scripts := fakeImageMountCommands(t, nil)
+	t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	imagePath := filepath.Join(t.TempDir(), "image.img")
+	if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	files := []inject.FileSpec{{Path: "user-data", Content: []byte("#cloud-config\n"), Mode: 0o644, RootFS: false}}
+	if err := InjectBootFiles(imagePath, files); err != nil {
+		t.Fatalf("InjectBootFiles: %v", err)
+	}
+	if got := readDumpedFile(t, dumpDir, "user-data"); got != "#cloud-config\n" {
+		t.Fatalf("dumped user-data = %q", got)
+	}
+}
+
+func TestInjectBootFilesNoopsWithoutBootFiles(t *testing.T) {
+	imagePath := filepath.Join(t.TempDir(), "image.img")
+	if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := InjectBootFiles(imagePath, []inject.FileSpec{{Path: "etc/metis/node.json", Content: []byte("{}"), Mode: 0o600, RootFS: true}}); err != nil {
+		t.Fatalf("InjectBootFiles root-only: %v", err)
+	}
+}
+
+func TestInjectBootFilesReportsMountFailures(t *testing.T) {
+	scripts := fakeImageMountCommands(t, map[string]string{"mount": `exit 9`})
+	t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	imagePath := filepath.Join(t.TempDir(), "image.img")
+	if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	err := InjectBootFiles(imagePath, []inject.FileSpec{{Path: "user-data", Content: []byte("boot"), Mode: 0o644, RootFS: false}})
+	if err == nil {
+		t.Fatal("expected mount failure")
+	}
+}
+
+func TestInjectImageWithFakes(t *testing.T) {
+	rootScripts := fakeRootfsCommands(t, true)
+	mountScripts := fakeImageMountCommands(t, nil)
+	t.Setenv("PATH", rootScripts+string(os.PathListSeparator)+mountScripts+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	dumpDir := filepath.Join(t.TempDir(), "dump")
+	t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir)
+	imagePath := filepath.Join(t.TempDir(), "image.img")
+	if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	files := []inject.FileSpec{
+		{Path: "etc/metis/firstboot.env", Content: []byte(`METIS_HOSTNAME='titan-13'
+`), Mode: 0o600, RootFS: true},
+		{Path: "user-data", Content: []byte(`#cloud-config
+`), Mode: 0o644, RootFS: false},
+	}
+	if err := InjectImage(imagePath, files); err != nil {
+		t.Fatalf("InjectImage: %v", err)
+	}
+	if got := readDumpedFile(t, dumpDir, "user-data"); got != `#cloud-config
+` {
+		t.Fatalf("dumped user-data = %q", got)
+	}
+}
+
+func readDumpedFile(t *testing.T, dumpDir, fileName string) string {
+	t.Helper()
+	var content string
+	err := filepath.Walk(dumpDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || filepath.Base(path) != fileName {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		content = string(data)
+		return filepath.SkipAll
+	})
+	if err != nil {
+		t.Fatalf("walk dump dir: %v", err)
+	}
+	if content == "" {
+		t.Fatalf("did not find %s under %s", fileName, dumpDir)
+	}
+	return content
+}
+
+func fakeImageMountCommands(t *testing.T, overrides map[string]string) string {
+	t.Helper()
+	dir := t.TempDir()
+	scripts := map[string]string{
+		"losetup": `if [[ "${1:-}" == "-Pf" && "${2:-}" == "--show" ]]; then
+  printf '/dev/loop9\n'
+  exit 0
+fi
+exit 0`,
+		"mount": `mkdir -p "${2:-}"
+exit 0`,
+		"umount": `target="${1:-}"
+dump="${METIS_FAKE_UMOUNT_DUMP_DIR:-}"
+if [[ -n "$dump" && -d "$target" ]]; then
+  base="$(basename "$target")"
+  mkdir -p "$dump/$base"
+  cp -a "$target"/. "$dump/$base/" 2>/dev/null || true
+fi
+exit 0`,
+	}
+	for name, body := range overrides {
+		scripts[name] = body
+	}
+	for name, body := range scripts {
+		path := filepath.Join(dir, name)
+		if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -eu\n"+body+"\n"), 0o755); err != nil {
+			t.Fatalf("write %s: %v", name, err)
+		}
+	}
+	return dir
+}
+
 func fakeRootfsCommands(t *testing.T, includeLinux bool) string {
 	t.Helper()
 	dir := t.TempDir()

pkg/plan/image_build.go

@@ -35,8 +35,8 @@ func BuildImageFile(ctx context.Context, inv *inventory.Inventory, nodeName, cac
 	if err != nil {
 		return fmt.Errorf("resolve files: %w", err)
 	}
-	if err := image.InjectRootFS(output, files); err != nil {
+	if err := image.InjectImage(output, files); err != nil {
-		return fmt.Errorf("inject rootfs: %w", err)
+		return fmt.Errorf("inject image: %w", err)
 	}
 	return nil
 }

pkg/plan/workflow_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"metis/pkg/inventory"
@@ -78,7 +79,8 @@ func TestExecuteAndBuildImageFileWithFakes(t *testing.T) {
 
 func TestBuildImageFileMaterializesRootFS(t *testing.T) {
 	rootTools := fakeRootfsTools(t)
-	t.Setenv("PATH", rootTools+string(os.PathListSeparator)+os.Getenv("PATH"))
+	mountTools := fakeMountTools(t)
+	t.Setenv("PATH", rootTools+string(os.PathListSeparator)+mountTools+string(os.PathListSeparator)+os.Getenv("PATH"))
 
 	dir := t.TempDir()
 	rawImage := filepath.Join(dir, "base.img")
@@ -104,6 +106,8 @@ func TestBuildImageFileMaterializesRootFS(t *testing.T) {
 			SSHAuthorized: []string{"ssh-ed25519 AAA"},
 		}},
 	}
+	dumpDir := filepath.Join(dir, "dump")
+	t.Setenv("METIS_FAKE_UMOUNT_DUMP_DIR", dumpDir)
 	out := filepath.Join(dir, "output.img")
 	if err := BuildImageFile(context.Background(), inv, "n1", filepath.Join(dir, "cache"), out); err != nil {
 		t.Fatalf("BuildImageFile: %v", err)
@@ -111,6 +115,10 @@ func TestBuildImageFileMaterializesRootFS(t *testing.T) {
 	if _, err := os.Stat(out); err != nil {
 		t.Fatalf("expected output image: %v", err)
 	}
+	userData := findDumpedFileContent(t, dumpDir, "user-data")
+	if !strings.Contains(userData, "metis-apply-node-identity.sh") {
+		t.Fatalf("expected boot cloud-init hook in user-data, got: %s", userData)
+	}
 }
 
 func TestMaybeInjectNoopsWhenEnvUnset(t *testing.T) {
@@ -183,11 +191,45 @@ func fakeMountTools(t *testing.T) string {
   exit 0
 fi
 exit 0`)
-	write("mount", `exit 0`)
+	write("mount", `mkdir -p "${2:-}"
-	write("umount", `exit 0`)
+exit 0`)
+	write("umount", `target="${1:-}"
+dump="${METIS_FAKE_UMOUNT_DUMP_DIR:-}"
+if [[ -n "$dump" && -d "$target" ]]; then
+  base="$(basename "$target")"
+  mkdir -p "$dump/$base"
+  cp -a "$target"/. "$dump/$base/" 2>/dev/null || true
+fi
+exit 0`)
 	return dir
 }
 
+func findDumpedFileContent(t *testing.T, dumpDir, fileName string) string {
+	t.Helper()
+	var content string
+	err := filepath.Walk(dumpDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || filepath.Base(path) != fileName {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		content = string(data)
+		return filepath.SkipAll
+	})
+	if err != nil {
+		t.Fatalf("walk dumped files: %v", err)
+	}
+	if content == "" {
+		t.Fatalf("did not find dumped %s under %s", fileName, dumpDir)
+	}
+	return content
+}
+
 func imageChecksum(t *testing.T, path string) string {
 	t.Helper()
 	data, err := os.ReadFile(path)

pkg/service/remote_helpers.go

@@ -284,19 +284,25 @@ func (a *App) remoteBuildPodSpec(name, host, image, node, nodeHostname, artifact
 							"--harbor-registry", a.settings.HarborRegistry,
 						),
 					},
-					"securityContext": map[string]any{"runAsUser": 0, "runAsGroup": 0},
+					"securityContext": map[string]any{"privileged": true, "runAsUser": 0, "runAsGroup": 0},
 					"env":             desiredEnv,
 					"envFrom": []map[string]any{
 						{"configMapRef": map[string]any{"name": "metis"}},
 					},
 					"volumeMounts": []map[string]any{
 						{"name": "workspace", "mountPath": "/workspace"},
+						{"name": "host-dev", "mountPath": "/dev"},
+						{"name": "host-sys", "mountPath": "/sys", "readOnly": true},
+						{"name": "host-udev", "mountPath": "/run/udev", "readOnly": true},
 					},
 				},
 			},
 			"imagePullSecrets": []map[string]string{{"name": "harbor-regcred"}},
 			"volumes": []map[string]any{
 				{"name": "workspace", "hostPath": map[string]any{"path": workspaceHostPath, "type": "DirectoryOrCreate"}},
+				{"name": "host-dev", "hostPath": map[string]any{"path": "/dev"}},
+				{"name": "host-sys", "hostPath": map[string]any{"path": "/sys"}},
+				{"name": "host-udev", "hostPath": map[string]any{"path": "/run/udev"}},
 			},
 		},
 	}

pkg/service/remote_helpers_test.go

@@ -277,12 +277,18 @@ func TestRemoteWorkspaceAndHostTmpPathsPreferUsbScratchRoots(t *testing.T) {
 		t.Fatalf("expected node password exports in vault template: %#v", metadataAnnotations)
 	}
 	buildSecurity := buildContainer["securityContext"].(map[string]any)
+	if got := buildSecurity["privileged"]; got != true {
+		t.Fatalf("build privileged = %v", got)
+	}
 	if got := buildSecurity["runAsUser"]; got != 0 {
 		t.Fatalf("build runAsUser = %v", got)
 	}
 	if got := buildSecurity["runAsGroup"]; got != 0 {
 		t.Fatalf("build runAsGroup = %v", got)
 	}
+	if got := buildVolumes[1]["hostPath"].(map[string]any)["path"]; got != "/dev" {
+		t.Fatalf("build host-dev hostPath = %v", got)
+	}
 
 	flashSpec := app.remoteFlashPodSpec("metis-flash-123", "titan-04", "runner:arm64", "titan-10", hostTmpDevicePath, "registry.example/metis/titan-10")
 	flashVolumes := flashSpec["spec"].(map[string]any)["volumes"].([]map[string]any)