runtime(metis): expose rootfs rewrite progress

cmd/metis/remote_cmd.go

@@ -95,9 +95,14 @@ func remoteBuildCmd(args []string) {
 		image.RootFSProgressWritingFiles:     {Stage: "build", ProgressPct: 50, Message: fmt.Sprintf("Injecting node-specific files into the root filesystem for %s", *node)},
 		image.RootFSProgressReplacing:        {Stage: "build", ProgressPct: 56, Message: fmt.Sprintf("Replacing the root partition inside the replacement image for %s", *node)},
 	}
-	if err := image.InjectRootFSWithProgress(output, files, func(step string) {
+	replaceEmitter := newProgressEmitter("build", 56, 58, fmt.Sprintf("Replacing the root partition inside the replacement image for %s", *node), true)
-		if update, ok := rootfsProgress[step]; ok {
+	if err := image.InjectRootFSWithProgress(output, files, func(update image.RootFSProgressUpdate) {
-			emitProgress(update)
+		if update.Step == image.RootFSProgressReplacing && update.TotalBytes > 0 {
+			replaceEmitter(update.WrittenBytes, update.TotalBytes)
+			return
+		}
+		if progressUpdate, ok := rootfsProgress[update.Step]; ok {
+			emitProgress(progressUpdate)
 		}
 	}); err != nil {
 		fatalf("inject rootfs: %v", err)
@@ -144,6 +149,7 @@ func remoteBuildCmd(args []string) {
 	if err := orasTag(taggedRef, "latest"); err != nil {
 		fatalf("oras tag latest: %v", err)
 	}
+	emitStageProgress("build", 80, fmt.Sprintf("Published the replacement image for %s to Harbor", *node))
 
 	summary := service.ArtifactSummary{
 		Node:       *node,

cmd/metis/remote_cmd_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"strings"
@@ -33,6 +34,20 @@ func TestOrasPushInvocationUsesRelativeWorkspacePaths(t *testing.T) {
 	}
 }
 
+func TestWriteStructuredResultFatalOnMarshalError(t *testing.T) {
+	oldFatalf := fatalf
+	fatalf = func(format string, args ...any) {
+		panic(fmt.Sprintf(format, args...))
+	}
+	defer func() { fatalf = oldFatalf }()
+	defer func() {
+		if got := recover(); got == nil || !strings.Contains(fmt.Sprint(got), "encode result") {
+			t.Fatalf("unexpected panic: %v", got)
+		}
+	}()
+	writeStructuredResult(make(chan int))
+}
+
 func TestHumanHostPathMapsMountedTmpBackToHostTmp(t *testing.T) {
 	if got := humanHostPath("/host-tmp/metis-flash-test"); got != "/tmp/metis-flash-test" {
 		t.Fatalf("expected /tmp/metis-flash-test, got %q", got)

cmd/metis/remote_edge_test.go

@@ -268,6 +268,30 @@ func TestPlanBurnAndRemoteBuildFatalStageEdges(t *testing.T) {
 			remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "login-fail"), "--cache", filepath.Join(root, "cache-login")})
 		})
 	})
+	t.Run("remote build oras push failure", func(t *testing.T) {
+		tools := remoteBuildToolDir(t, baseImage, `cp "${@: -1}" "${@: -1}.xz"`, `case "${1:-}" in
+  login|pull) exit 0 ;;
+  push) printf 'push failed' >&2; exit 6 ;;
+  tag) exit 0 ;;
+esac
+exit 0`)
+		t.Setenv("PATH", tools+string(os.PathListSeparator)+os.Getenv("PATH"))
+		expectCommandFatal(t, "oras push", func() {
+			remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "push-fail"), "--cache", filepath.Join(root, "cache-push"), "--harbor-username", "admin", "--harbor-password", "pw"})
+		})
+	})
+
+	t.Run("remote build oras tag failure", func(t *testing.T) {
+		tools := remoteBuildToolDir(t, baseImage, `cp "${@: -1}" "${@: -1}.xz"`, `case "${1:-}" in
+  login|pull|push) exit 0 ;;
+  tag) printf 'tag failed' >&2; exit 7 ;;
+esac
+exit 0`)
+		t.Setenv("PATH", tools+string(os.PathListSeparator)+os.Getenv("PATH"))
+		expectCommandFatal(t, "oras tag latest", func() {
+			remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "tag-fail"), "--cache", filepath.Join(root, "cache-tag"), "--harbor-username", "admin", "--harbor-password", "pw"})
+		})
+	})
 }
 
 func TestRemoteFlashFatalAndDeviceFlushEdges(t *testing.T) {

pkg/image/rootfs.go

@@ -29,9 +29,16 @@ type partitionTablePart struct {
 	Type  string `json:"type"`
 }
 
-// RootFSProgressFunc receives coarse-grained step names while Metis rewrites a
+// RootFSProgressUpdate carries coarse-grained step changes and optional byte
-// Linux root filesystem inside a raw image.
+// counters while Metis rewrites a Linux root filesystem inside a raw image.
-type RootFSProgressFunc func(step string)
+type RootFSProgressUpdate struct {
+	Step         string
+	WrittenBytes int64
+	TotalBytes   int64
+}
+
+// RootFSProgressFunc receives RootFS progress updates during image rewriting.
+type RootFSProgressFunc func(update RootFSProgressUpdate)
 
 const (
 	RootFSProgressFindingPartition = "finding-partition"
@@ -58,34 +65,40 @@ func InjectRootFSWithProgress(imagePath string, files []inject.FileSpec, progres
 		return nil
 	}
 
-	emitRootFSProgress(progress, RootFSProgressFindingPartition)
+	emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressFindingPartition})
 	part, sectorSize, err := findLinuxPartition(imagePath)
 	if err != nil {
 		return err
 	}
 
-	workDir, err := os.MkdirTemp("", "metis-rootfs-")
+	workDir, err := mkdirTempNearPath(imagePath, "metis-rootfs-")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(workDir)
 
 	rootImage := filepath.Join(workDir, "root.ext4")
-	emitRootFSProgress(progress, RootFSProgressExtracting)
+	emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressExtracting})
 	if err := extractPartition(imagePath, rootImage, part, sectorSize); err != nil {
 		return err
 	}
-	emitRootFSProgress(progress, RootFSProgressWritingFiles)
+	emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressWritingFiles})
 	if err := writeExt4Files(rootImage, rootFiles); err != nil {
 		return err
 	}
-	emitRootFSProgress(progress, RootFSProgressReplacing)
+	emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressReplacing})
-	return replacePartition(imagePath, rootImage, part, sectorSize)
+	return replacePartition(imagePath, rootImage, part, sectorSize, func(written, total int64) {
+		emitRootFSProgress(progress, RootFSProgressUpdate{
+			Step:         RootFSProgressReplacing,
+			WrittenBytes: written,
+			TotalBytes:   total,
+		})
+	})
 }
 
-func emitRootFSProgress(progress RootFSProgressFunc, step string) {
+func emitRootFSProgress(progress RootFSProgressFunc, update RootFSProgressUpdate) {
 	if progress != nil {
-		progress(step)
+		progress(update)
 	}
 }
 
@@ -144,7 +157,7 @@ func extractPartition(imagePath, outPath string, part partitionTablePart, sector
 	return out.Sync()
 }
 
-func replacePartition(imagePath, rootImage string, part partitionTablePart, sectorSize uint64) error {
+func replacePartition(imagePath, rootImage string, part partitionTablePart, sectorSize uint64, progress func(written, total int64)) error {
 	expectedSize := int64(part.Size * sectorSize)
 	info, err := os.Stat(rootImage)
 	if err != nil {
@@ -168,14 +181,14 @@ func replacePartition(imagePath, rootImage string, part partitionTablePart, sect
 	if _, err := out.Seek(int64(part.Start*sectorSize), io.SeekStart); err != nil {
 		return err
 	}
-	if _, err := io.Copy(out, in); err != nil {
+	if _, err := copyWithProgress(out, in, expectedSize, progress); err != nil {
 		return fmt.Errorf("write root partition: %w", err)
 	}
 	return out.Sync()
 }
 
 func writeExt4Files(fsPath string, files []inject.FileSpec) error {
-	workDir, err := os.MkdirTemp("", "metis-ext4-")
+	workDir, err := mkdirTempNearPath(fsPath, "metis-ext4-")
 	if err != nil {
 		return err
 	}
@@ -243,6 +256,47 @@ func writeExt4Files(fsPath string, files []inject.FileSpec) error {
 	return nil
 }
 
+func mkdirTempNearPath(targetPath, pattern string) (string, error) {
+	parent := strings.TrimSpace(os.Getenv("METIS_ROOTFS_TMP_DIR"))
+	if parent == "" && strings.TrimSpace(targetPath) != "" {
+		parent = filepath.Join(filepath.Dir(targetPath), ".metis-tmp")
+	}
+	if parent == "" {
+		return os.MkdirTemp("", pattern)
+	}
+	if err := os.MkdirAll(parent, 0o755); err != nil {
+		return "", err
+	}
+	return os.MkdirTemp(parent, pattern)
+}
+
+func copyWithProgress(dst io.Writer, src io.Reader, total int64, progress func(written, total int64)) (int64, error) {
+	buf := make([]byte, 2*1024*1024)
+	var written int64
+	for {
+		nr, er := src.Read(buf)
+		if nr > 0 {
+			nw, ew := dst.Write(buf[:nr])
+			written += int64(nw)
+			if progress != nil {
+				progress(written, total)
+			}
+			if ew != nil {
+				return written, ew
+			}
+			if nw != nr {
+				return written, io.ErrShortWrite
+			}
+		}
+		if er != nil {
+			if er == io.EOF {
+				return written, nil
+			}
+			return written, er
+		}
+	}
+}
+
 func verifyExt4File(fsPath string, file inject.FileSpec, workDir string) error {
 	destPath := "/" + strings.TrimPrefix(filepath.ToSlash(file.Path), "/")
 	statOut, err := exec.Command("debugfs", "-R", "stat "+destPath, fsPath).CombinedOutput()

pkg/image/rootfs_progress_test.go

@@ -0,0 +1,161 @@
+package image
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"metis/pkg/inject"
+)
+
+type shortWriteOnly struct{}
+
+func (shortWriteOnly) Write(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	return len(p) - 1, nil
+}
+
+type failAfterFirstRead struct{ done bool }
+
+func (r *failAfterFirstRead) Read(p []byte) (int, error) {
+	if r.done {
+		return 0, io.ErrUnexpectedEOF
+	}
+	r.done = true
+	copy(p, []byte("ab"))
+	return 2, nil
+}
+
+func TestInjectRootFSWithFakesReportsReplacingByteProgress(t *testing.T) {
+	scripts := fakeRootfsCommands(t, true)
+	t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	imagePath := filepath.Join(t.TempDir(), "image.img")
+	if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	files := []inject.FileSpec{{Path: "etc/metis/firstboot.env", Content: []byte("METIS_HOSTNAME='titan-13'\n"), Mode: 0o600, RootFS: true}}
+	var replacing []RootFSProgressUpdate
+	if err := InjectRootFSWithProgress(imagePath, files, func(update RootFSProgressUpdate) {
+		if update.Step == RootFSProgressReplacing && update.TotalBytes > 0 {
+			replacing = append(replacing, update)
+		}
+	}); err != nil {
+		t.Fatalf("InjectRootFSWithProgress: %v", err)
+	}
+	if len(replacing) == 0 {
+		t.Fatal("expected replacing progress updates")
+	}
+	last := replacing[len(replacing)-1]
+	if last.WrittenBytes != last.TotalBytes || last.TotalBytes != 1024 {
+		t.Fatalf("unexpected replacing progress: %+v", last)
+	}
+}
+
+func TestMkdirTempNearPathPrefersImageAdjacentScratch(t *testing.T) {
+	imagePath := filepath.Join(t.TempDir(), "build", "node.img")
+	if err := os.MkdirAll(filepath.Dir(imagePath), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	workDir, err := mkdirTempNearPath(imagePath, "metis-rootfs-")
+	if err != nil {
+		t.Fatalf("mkdirTempNearPath: %v", err)
+	}
+	defer os.RemoveAll(workDir)
+	if got, want := filepath.Base(filepath.Dir(workDir)), ".metis-tmp"; got != want {
+		t.Fatalf("temp parent = %q want %q", got, want)
+	}
+}
+
+func TestCopyWithProgressReportsByteCounts(t *testing.T) {
+	var dst bytes.Buffer
+	var updates []RootFSProgressUpdate
+	src := bytes.NewReader([]byte("hello world"))
+	written, err := copyWithProgress(&dst, src, int64(src.Len()), func(written, total int64) {
+		updates = append(updates, RootFSProgressUpdate{Step: RootFSProgressReplacing, WrittenBytes: written, TotalBytes: total})
+	})
+	if err != nil {
+		t.Fatalf("copyWithProgress: %v", err)
+	}
+	if written != int64(len("hello world")) {
+		t.Fatalf("written = %d", written)
+	}
+	if dst.String() != "hello world" {
+		t.Fatalf("unexpected dst content: %q", dst.String())
+	}
+	if len(updates) == 0 || updates[len(updates)-1].WrittenBytes != updates[len(updates)-1].TotalBytes {
+		t.Fatalf("expected final byte-complete update, got %#v", updates)
+	}
+}
+
+func TestReplacePartitionWritesDataAndReportsProgress(t *testing.T) {
+	dir := t.TempDir()
+	imagePath := filepath.Join(dir, "image.img")
+	rootPath := filepath.Join(dir, "root.ext4")
+	imageBytes := make([]byte, 1024)
+	rootBytes := bytes.Repeat([]byte("r"), 512)
+	if err := os.WriteFile(imagePath, imageBytes, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(rootPath, rootBytes, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var updates []RootFSProgressUpdate
+	if err := replacePartition(imagePath, rootPath, partitionTablePart{Start: 1, Size: 1}, 512, func(written, total int64) {
+		updates = append(updates, RootFSProgressUpdate{Step: RootFSProgressReplacing, WrittenBytes: written, TotalBytes: total})
+	}); err != nil {
+		t.Fatalf("replacePartition: %v", err)
+	}
+	got, err := os.ReadFile(imagePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(got[512:], rootBytes) {
+		t.Fatalf("partition bytes were not replaced: %q", got[512:])
+	}
+	if len(updates) == 0 || updates[len(updates)-1].WrittenBytes != 512 || updates[len(updates)-1].TotalBytes != 512 {
+		t.Fatalf("expected final replace progress update, got %#v", updates)
+	}
+}
+
+func TestMkdirTempNearPathHonorsOverrideAndFallback(t *testing.T) {
+	t.Run("override", func(t *testing.T) {
+		override := filepath.Join(t.TempDir(), "override")
+		t.Setenv("METIS_ROOTFS_TMP_DIR", override)
+		workDir, err := mkdirTempNearPath(filepath.Join(t.TempDir(), "build", "node.img"), "metis-rootfs-")
+		if err != nil {
+			t.Fatalf("mkdirTempNearPath override: %v", err)
+		}
+		defer os.RemoveAll(workDir)
+		if got := filepath.Dir(workDir); got != override {
+			t.Fatalf("override parent = %q want %q", got, override)
+		}
+	})
+	t.Run("fallback tempdir", func(t *testing.T) {
+		workDir, err := mkdirTempNearPath("", "metis-rootfs-")
+		if err != nil {
+			t.Fatalf("mkdirTempNearPath fallback: %v", err)
+		}
+		defer os.RemoveAll(workDir)
+		if workDir == "" {
+			t.Fatal("expected fallback temp dir path")
+		}
+	})
+}
+
+func TestCopyWithProgressPropagatesReaderAndWriterErrors(t *testing.T) {
+	t.Run("short write", func(t *testing.T) {
+		if _, err := copyWithProgress(shortWriteOnly{}, bytes.NewReader([]byte("abc")), 3, nil); err != io.ErrShortWrite {
+			t.Fatalf("expected short write, got %v", err)
+		}
+	})
+	t.Run("reader error", func(t *testing.T) {
+		if _, err := copyWithProgress(&bytes.Buffer{}, &failAfterFirstRead{}, 4, nil); err != io.ErrUnexpectedEOF {
+			t.Fatalf("expected reader error, got %v", err)
+		}
+	})
+}

pkg/image/rootfs_test.go

@@ -96,7 +96,7 @@ func TestRootfsErrorBranches(t *testing.T) {
 	if err := os.WriteFile(dst, make([]byte, 512), 0o644); err != nil {
 		t.Fatal(err)
 	}
-	if err := replacePartition(src, dst, part, 512); err == nil {
+	if err := replacePartition(src, dst, part, 512, nil); err == nil {
 		t.Fatal("expected replacePartition size mismatch")
 	}
 }
@@ -113,8 +113,8 @@ func TestRootfsAdditionalTopLevelErrorBranches(t *testing.T) {
 		scripts := fakeImageCommandDir(t, map[string]string{"sfdisk": `exit 3`})
 		t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH"))
 		var steps []string
-		err := InjectRootFSWithProgress(imagePath, rootFiles, func(step string) {
+		err := InjectRootFSWithProgress(imagePath, rootFiles, func(update RootFSProgressUpdate) {
-			steps = append(steps, step)
+			steps = append(steps, update.Step)
 		})
 		if err == nil || len(steps) != 1 || steps[0] != RootFSProgressFindingPartition {
 			t.Fatalf("expected partition error after progress, steps=%v err=%v", steps, err)
@@ -149,7 +149,7 @@ JSON`})
 		if err := os.WriteFile(tmpFile, []byte("not-a-dir"), 0o644); err != nil {
 			t.Fatal(err)
 		}
-		t.Setenv("TMPDIR", tmpFile)
+		t.Setenv("METIS_ROOTFS_TMP_DIR", tmpFile)
 		if err := InjectRootFSWithProgress(imagePath, rootFiles, nil); err == nil {
 			t.Fatal("expected rootfs temp-dir failure")
 		}
@@ -206,18 +206,18 @@ func TestRootfsDirectHelperErrorBranches(t *testing.T) {
 		t.Fatal("expected extract destination create failure")
 	}
 
-	if err := replacePartition(src, filepath.Join(dir, "missing-root.img"), partitionTablePart{Start: 0, Size: 1}, 512); err == nil {
+	if err := replacePartition(src, filepath.Join(dir, "missing-root.img"), partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil {
 		t.Fatal("expected missing root image failure")
 	}
 	root := filepath.Join(dir, "root.img")
 	if err := os.WriteFile(root, make([]byte, 512), 0o644); err != nil {
 		t.Fatal(err)
 	}
-	if err := replacePartition(filepath.Join(dir, "missing-image.img"), root, partitionTablePart{Start: 0, Size: 1}, 512); err == nil {
+	if err := replacePartition(filepath.Join(dir, "missing-image.img"), root, partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil {
 		t.Fatal("expected destination image open failure")
 	}
 	if _, err := os.Stat("/dev/full"); err == nil {
-		if err := replacePartition("/dev/full", root, partitionTablePart{Start: 0, Size: 1}, 512); err == nil {
+		if err := replacePartition("/dev/full", root, partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil {
 			t.Fatal("expected /dev/full replace failure")
 		}
 	}
@@ -226,7 +226,7 @@ func TestRootfsDirectHelperErrorBranches(t *testing.T) {
 	if err := os.WriteFile(tmpFile, []byte("not-a-dir"), 0o644); err != nil {
 		t.Fatal(err)
 	}
-	t.Setenv("TMPDIR", tmpFile)
+	t.Setenv("METIS_ROOTFS_TMP_DIR", tmpFile)
 	if err := writeExt4Files(src, []inject.FileSpec{{Path: "etc/node.json", Content: []byte("{}"), Mode: 0o600, RootFS: true}}); err == nil {
 		t.Fatal("expected writeExt4Files temp-dir failure")
 	}