From bf9c514fd06eb2ec2b522ae19f33fede9cbe4add Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 24 Apr 2026 01:58:29 -0300 Subject: [PATCH] runtime(metis): expose rootfs rewrite progress --- cmd/metis/remote_cmd.go | 12 ++- cmd/metis/remote_cmd_test.go | 15 +++ cmd/metis/remote_edge_test.go | 24 +++++ pkg/image/rootfs.go | 82 ++++++++++++--- pkg/image/rootfs_progress_test.go | 161 ++++++++++++++++++++++++++++++ pkg/image/rootfs_test.go | 16 +-- 6 files changed, 285 insertions(+), 25 deletions(-) create mode 100644 pkg/image/rootfs_progress_test.go diff --git a/cmd/metis/remote_cmd.go b/cmd/metis/remote_cmd.go index 29b5523..f1f33b8 100644 --- a/cmd/metis/remote_cmd.go +++ b/cmd/metis/remote_cmd.go @@ -95,9 +95,14 @@ func remoteBuildCmd(args []string) { image.RootFSProgressWritingFiles: {Stage: "build", ProgressPct: 50, Message: fmt.Sprintf("Injecting node-specific files into the root filesystem for %s", *node)}, image.RootFSProgressReplacing: {Stage: "build", ProgressPct: 56, Message: fmt.Sprintf("Replacing the root partition inside the replacement image for %s", *node)}, } - if err := image.InjectRootFSWithProgress(output, files, func(step string) { - if update, ok := rootfsProgress[step]; ok { - emitProgress(update) + replaceEmitter := newProgressEmitter("build", 56, 58, fmt.Sprintf("Replacing the root partition inside the replacement image for %s", *node), true) + if err := image.InjectRootFSWithProgress(output, files, func(update image.RootFSProgressUpdate) { + if update.Step == image.RootFSProgressReplacing && update.TotalBytes > 0 { + replaceEmitter(update.WrittenBytes, update.TotalBytes) + return + } + if progressUpdate, ok := rootfsProgress[update.Step]; ok { + emitProgress(progressUpdate) } }); err != nil { fatalf("inject rootfs: %v", err) @@ -144,6 +149,7 @@ func remoteBuildCmd(args []string) { if err := orasTag(taggedRef, "latest"); err != nil { fatalf("oras tag latest: %v", err) } + emitStageProgress("build", 80, fmt.Sprintf("Published the replacement image for %s to Harbor", *node)) summary := service.ArtifactSummary{ Node: *node, diff --git a/cmd/metis/remote_cmd_test.go b/cmd/metis/remote_cmd_test.go index 4f70dbb..1ebe030 100644 --- a/cmd/metis/remote_cmd_test.go +++ b/cmd/metis/remote_cmd_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "fmt" "io" "os" "strings" @@ -33,6 +34,20 @@ func TestOrasPushInvocationUsesRelativeWorkspacePaths(t *testing.T) { } } +func TestWriteStructuredResultFatalOnMarshalError(t *testing.T) { + oldFatalf := fatalf + fatalf = func(format string, args ...any) { + panic(fmt.Sprintf(format, args...)) + } + defer func() { fatalf = oldFatalf }() + defer func() { + if got := recover(); got == nil || !strings.Contains(fmt.Sprint(got), "encode result") { + t.Fatalf("unexpected panic: %v", got) + } + }() + writeStructuredResult(make(chan int)) +} + func TestHumanHostPathMapsMountedTmpBackToHostTmp(t *testing.T) { if got := humanHostPath("/host-tmp/metis-flash-test"); got != "/tmp/metis-flash-test" { t.Fatalf("expected /tmp/metis-flash-test, got %q", got) diff --git a/cmd/metis/remote_edge_test.go b/cmd/metis/remote_edge_test.go index 6deb0e2..5207a12 100644 --- a/cmd/metis/remote_edge_test.go +++ b/cmd/metis/remote_edge_test.go @@ -268,6 +268,30 @@ func TestPlanBurnAndRemoteBuildFatalStageEdges(t *testing.T) { remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "login-fail"), "--cache", filepath.Join(root, "cache-login")}) }) }) + t.Run("remote build oras push failure", func(t *testing.T) { + tools := remoteBuildToolDir(t, baseImage, `cp "${@: -1}" "${@: -1}.xz"`, `case "${1:-}" in + login|pull) exit 0 ;; + push) printf 'push failed' >&2; exit 6 ;; + tag) exit 0 ;; +esac +exit 0`) + t.Setenv("PATH", tools+string(os.PathListSeparator)+os.Getenv("PATH")) + expectCommandFatal(t, "oras push", func() { + remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "push-fail"), "--cache", filepath.Join(root, "cache-push"), "--harbor-username", "admin", "--harbor-password", "pw"}) + }) + }) + + t.Run("remote build oras tag failure", func(t *testing.T) { + tools := remoteBuildToolDir(t, baseImage, `cp "${@: -1}" "${@: -1}.xz"`, `case "${1:-}" in + login|pull|push) exit 0 ;; + tag) printf 'tag failed' >&2; exit 7 ;; +esac +exit 0`) + t.Setenv("PATH", tools+string(os.PathListSeparator)+os.Getenv("PATH")) + expectCommandFatal(t, "oras tag latest", func() { + remoteBuildCmd([]string{"--inventory", invPath, "--node", "titan-15", "--artifact-ref", "registry.example/metis/titan-15", "--build-tag", "build-1", "--work-dir", filepath.Join(root, "tag-fail"), "--cache", filepath.Join(root, "cache-tag"), "--harbor-username", "admin", "--harbor-password", "pw"}) + }) + }) } func TestRemoteFlashFatalAndDeviceFlushEdges(t *testing.T) { diff --git a/pkg/image/rootfs.go b/pkg/image/rootfs.go index f9efc99..c713a89 100644 --- a/pkg/image/rootfs.go +++ b/pkg/image/rootfs.go @@ -29,9 +29,16 @@ type partitionTablePart struct { Type string `json:"type"` } -// RootFSProgressFunc receives coarse-grained step names while Metis rewrites a -// Linux root filesystem inside a raw image. -type RootFSProgressFunc func(step string) +// RootFSProgressUpdate carries coarse-grained step changes and optional byte +// counters while Metis rewrites a Linux root filesystem inside a raw image. +type RootFSProgressUpdate struct { + Step string + WrittenBytes int64 + TotalBytes int64 +} + +// RootFSProgressFunc receives RootFS progress updates during image rewriting. +type RootFSProgressFunc func(update RootFSProgressUpdate) const ( RootFSProgressFindingPartition = "finding-partition" @@ -58,34 +65,40 @@ func InjectRootFSWithProgress(imagePath string, files []inject.FileSpec, progres return nil } - emitRootFSProgress(progress, RootFSProgressFindingPartition) + emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressFindingPartition}) part, sectorSize, err := findLinuxPartition(imagePath) if err != nil { return err } - workDir, err := os.MkdirTemp("", "metis-rootfs-") + workDir, err := mkdirTempNearPath(imagePath, "metis-rootfs-") if err != nil { return err } defer os.RemoveAll(workDir) rootImage := filepath.Join(workDir, "root.ext4") - emitRootFSProgress(progress, RootFSProgressExtracting) + emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressExtracting}) if err := extractPartition(imagePath, rootImage, part, sectorSize); err != nil { return err } - emitRootFSProgress(progress, RootFSProgressWritingFiles) + emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressWritingFiles}) if err := writeExt4Files(rootImage, rootFiles); err != nil { return err } - emitRootFSProgress(progress, RootFSProgressReplacing) - return replacePartition(imagePath, rootImage, part, sectorSize) + emitRootFSProgress(progress, RootFSProgressUpdate{Step: RootFSProgressReplacing}) + return replacePartition(imagePath, rootImage, part, sectorSize, func(written, total int64) { + emitRootFSProgress(progress, RootFSProgressUpdate{ + Step: RootFSProgressReplacing, + WrittenBytes: written, + TotalBytes: total, + }) + }) } -func emitRootFSProgress(progress RootFSProgressFunc, step string) { +func emitRootFSProgress(progress RootFSProgressFunc, update RootFSProgressUpdate) { if progress != nil { - progress(step) + progress(update) } } @@ -144,7 +157,7 @@ func extractPartition(imagePath, outPath string, part partitionTablePart, sector return out.Sync() } -func replacePartition(imagePath, rootImage string, part partitionTablePart, sectorSize uint64) error { +func replacePartition(imagePath, rootImage string, part partitionTablePart, sectorSize uint64, progress func(written, total int64)) error { expectedSize := int64(part.Size * sectorSize) info, err := os.Stat(rootImage) if err != nil { @@ -168,14 +181,14 @@ func replacePartition(imagePath, rootImage string, part partitionTablePart, sect if _, err := out.Seek(int64(part.Start*sectorSize), io.SeekStart); err != nil { return err } - if _, err := io.Copy(out, in); err != nil { + if _, err := copyWithProgress(out, in, expectedSize, progress); err != nil { return fmt.Errorf("write root partition: %w", err) } return out.Sync() } func writeExt4Files(fsPath string, files []inject.FileSpec) error { - workDir, err := os.MkdirTemp("", "metis-ext4-") + workDir, err := mkdirTempNearPath(fsPath, "metis-ext4-") if err != nil { return err } @@ -243,6 +256,47 @@ func writeExt4Files(fsPath string, files []inject.FileSpec) error { return nil } +func mkdirTempNearPath(targetPath, pattern string) (string, error) { + parent := strings.TrimSpace(os.Getenv("METIS_ROOTFS_TMP_DIR")) + if parent == "" && strings.TrimSpace(targetPath) != "" { + parent = filepath.Join(filepath.Dir(targetPath), ".metis-tmp") + } + if parent == "" { + return os.MkdirTemp("", pattern) + } + if err := os.MkdirAll(parent, 0o755); err != nil { + return "", err + } + return os.MkdirTemp(parent, pattern) +} + +func copyWithProgress(dst io.Writer, src io.Reader, total int64, progress func(written, total int64)) (int64, error) { + buf := make([]byte, 2*1024*1024) + var written int64 + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[:nr]) + written += int64(nw) + if progress != nil { + progress(written, total) + } + if ew != nil { + return written, ew + } + if nw != nr { + return written, io.ErrShortWrite + } + } + if er != nil { + if er == io.EOF { + return written, nil + } + return written, er + } + } +} + func verifyExt4File(fsPath string, file inject.FileSpec, workDir string) error { destPath := "/" + strings.TrimPrefix(filepath.ToSlash(file.Path), "/") statOut, err := exec.Command("debugfs", "-R", "stat "+destPath, fsPath).CombinedOutput() diff --git a/pkg/image/rootfs_progress_test.go b/pkg/image/rootfs_progress_test.go new file mode 100644 index 0000000..3415712 --- /dev/null +++ b/pkg/image/rootfs_progress_test.go @@ -0,0 +1,161 @@ +package image + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "metis/pkg/inject" +) + +type shortWriteOnly struct{} + +func (shortWriteOnly) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + return len(p) - 1, nil +} + +type failAfterFirstRead struct{ done bool } + +func (r *failAfterFirstRead) Read(p []byte) (int, error) { + if r.done { + return 0, io.ErrUnexpectedEOF + } + r.done = true + copy(p, []byte("ab")) + return 2, nil +} + +func TestInjectRootFSWithFakesReportsReplacingByteProgress(t *testing.T) { + scripts := fakeRootfsCommands(t, true) + t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH")) + + imagePath := filepath.Join(t.TempDir(), "image.img") + if err := os.WriteFile(imagePath, make([]byte, 4096), 0o644); err != nil { + t.Fatal(err) + } + files := []inject.FileSpec{{Path: "etc/metis/firstboot.env", Content: []byte("METIS_HOSTNAME='titan-13'\n"), Mode: 0o600, RootFS: true}} + var replacing []RootFSProgressUpdate + if err := InjectRootFSWithProgress(imagePath, files, func(update RootFSProgressUpdate) { + if update.Step == RootFSProgressReplacing && update.TotalBytes > 0 { + replacing = append(replacing, update) + } + }); err != nil { + t.Fatalf("InjectRootFSWithProgress: %v", err) + } + if len(replacing) == 0 { + t.Fatal("expected replacing progress updates") + } + last := replacing[len(replacing)-1] + if last.WrittenBytes != last.TotalBytes || last.TotalBytes != 1024 { + t.Fatalf("unexpected replacing progress: %+v", last) + } +} + +func TestMkdirTempNearPathPrefersImageAdjacentScratch(t *testing.T) { + imagePath := filepath.Join(t.TempDir(), "build", "node.img") + if err := os.MkdirAll(filepath.Dir(imagePath), 0o755); err != nil { + t.Fatal(err) + } + workDir, err := mkdirTempNearPath(imagePath, "metis-rootfs-") + if err != nil { + t.Fatalf("mkdirTempNearPath: %v", err) + } + defer os.RemoveAll(workDir) + if got, want := filepath.Base(filepath.Dir(workDir)), ".metis-tmp"; got != want { + t.Fatalf("temp parent = %q want %q", got, want) + } +} + +func TestCopyWithProgressReportsByteCounts(t *testing.T) { + var dst bytes.Buffer + var updates []RootFSProgressUpdate + src := bytes.NewReader([]byte("hello world")) + written, err := copyWithProgress(&dst, src, int64(src.Len()), func(written, total int64) { + updates = append(updates, RootFSProgressUpdate{Step: RootFSProgressReplacing, WrittenBytes: written, TotalBytes: total}) + }) + if err != nil { + t.Fatalf("copyWithProgress: %v", err) + } + if written != int64(len("hello world")) { + t.Fatalf("written = %d", written) + } + if dst.String() != "hello world" { + t.Fatalf("unexpected dst content: %q", dst.String()) + } + if len(updates) == 0 || updates[len(updates)-1].WrittenBytes != updates[len(updates)-1].TotalBytes { + t.Fatalf("expected final byte-complete update, got %#v", updates) + } +} + +func TestReplacePartitionWritesDataAndReportsProgress(t *testing.T) { + dir := t.TempDir() + imagePath := filepath.Join(dir, "image.img") + rootPath := filepath.Join(dir, "root.ext4") + imageBytes := make([]byte, 1024) + rootBytes := bytes.Repeat([]byte("r"), 512) + if err := os.WriteFile(imagePath, imageBytes, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rootPath, rootBytes, 0o644); err != nil { + t.Fatal(err) + } + var updates []RootFSProgressUpdate + if err := replacePartition(imagePath, rootPath, partitionTablePart{Start: 1, Size: 1}, 512, func(written, total int64) { + updates = append(updates, RootFSProgressUpdate{Step: RootFSProgressReplacing, WrittenBytes: written, TotalBytes: total}) + }); err != nil { + t.Fatalf("replacePartition: %v", err) + } + got, err := os.ReadFile(imagePath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got[512:], rootBytes) { + t.Fatalf("partition bytes were not replaced: %q", got[512:]) + } + if len(updates) == 0 || updates[len(updates)-1].WrittenBytes != 512 || updates[len(updates)-1].TotalBytes != 512 { + t.Fatalf("expected final replace progress update, got %#v", updates) + } +} + +func TestMkdirTempNearPathHonorsOverrideAndFallback(t *testing.T) { + t.Run("override", func(t *testing.T) { + override := filepath.Join(t.TempDir(), "override") + t.Setenv("METIS_ROOTFS_TMP_DIR", override) + workDir, err := mkdirTempNearPath(filepath.Join(t.TempDir(), "build", "node.img"), "metis-rootfs-") + if err != nil { + t.Fatalf("mkdirTempNearPath override: %v", err) + } + defer os.RemoveAll(workDir) + if got := filepath.Dir(workDir); got != override { + t.Fatalf("override parent = %q want %q", got, override) + } + }) + t.Run("fallback tempdir", func(t *testing.T) { + workDir, err := mkdirTempNearPath("", "metis-rootfs-") + if err != nil { + t.Fatalf("mkdirTempNearPath fallback: %v", err) + } + defer os.RemoveAll(workDir) + if workDir == "" { + t.Fatal("expected fallback temp dir path") + } + }) +} + +func TestCopyWithProgressPropagatesReaderAndWriterErrors(t *testing.T) { + t.Run("short write", func(t *testing.T) { + if _, err := copyWithProgress(shortWriteOnly{}, bytes.NewReader([]byte("abc")), 3, nil); err != io.ErrShortWrite { + t.Fatalf("expected short write, got %v", err) + } + }) + t.Run("reader error", func(t *testing.T) { + if _, err := copyWithProgress(&bytes.Buffer{}, &failAfterFirstRead{}, 4, nil); err != io.ErrUnexpectedEOF { + t.Fatalf("expected reader error, got %v", err) + } + }) +} diff --git a/pkg/image/rootfs_test.go b/pkg/image/rootfs_test.go index 1120cc7..7974594 100644 --- a/pkg/image/rootfs_test.go +++ b/pkg/image/rootfs_test.go @@ -96,7 +96,7 @@ func TestRootfsErrorBranches(t *testing.T) { if err := os.WriteFile(dst, make([]byte, 512), 0o644); err != nil { t.Fatal(err) } - if err := replacePartition(src, dst, part, 512); err == nil { + if err := replacePartition(src, dst, part, 512, nil); err == nil { t.Fatal("expected replacePartition size mismatch") } } @@ -113,8 +113,8 @@ func TestRootfsAdditionalTopLevelErrorBranches(t *testing.T) { scripts := fakeImageCommandDir(t, map[string]string{"sfdisk": `exit 3`}) t.Setenv("PATH", scripts+string(os.PathListSeparator)+os.Getenv("PATH")) var steps []string - err := InjectRootFSWithProgress(imagePath, rootFiles, func(step string) { - steps = append(steps, step) + err := InjectRootFSWithProgress(imagePath, rootFiles, func(update RootFSProgressUpdate) { + steps = append(steps, update.Step) }) if err == nil || len(steps) != 1 || steps[0] != RootFSProgressFindingPartition { t.Fatalf("expected partition error after progress, steps=%v err=%v", steps, err) @@ -149,7 +149,7 @@ JSON`}) if err := os.WriteFile(tmpFile, []byte("not-a-dir"), 0o644); err != nil { t.Fatal(err) } - t.Setenv("TMPDIR", tmpFile) + t.Setenv("METIS_ROOTFS_TMP_DIR", tmpFile) if err := InjectRootFSWithProgress(imagePath, rootFiles, nil); err == nil { t.Fatal("expected rootfs temp-dir failure") } @@ -206,18 +206,18 @@ func TestRootfsDirectHelperErrorBranches(t *testing.T) { t.Fatal("expected extract destination create failure") } - if err := replacePartition(src, filepath.Join(dir, "missing-root.img"), partitionTablePart{Start: 0, Size: 1}, 512); err == nil { + if err := replacePartition(src, filepath.Join(dir, "missing-root.img"), partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil { t.Fatal("expected missing root image failure") } root := filepath.Join(dir, "root.img") if err := os.WriteFile(root, make([]byte, 512), 0o644); err != nil { t.Fatal(err) } - if err := replacePartition(filepath.Join(dir, "missing-image.img"), root, partitionTablePart{Start: 0, Size: 1}, 512); err == nil { + if err := replacePartition(filepath.Join(dir, "missing-image.img"), root, partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil { t.Fatal("expected destination image open failure") } if _, err := os.Stat("/dev/full"); err == nil { - if err := replacePartition("/dev/full", root, partitionTablePart{Start: 0, Size: 1}, 512); err == nil { + if err := replacePartition("/dev/full", root, partitionTablePart{Start: 0, Size: 1}, 512, nil); err == nil { t.Fatal("expected /dev/full replace failure") } } @@ -226,7 +226,7 @@ func TestRootfsDirectHelperErrorBranches(t *testing.T) { if err := os.WriteFile(tmpFile, []byte("not-a-dir"), 0o644); err != nil { t.Fatal(err) } - t.Setenv("TMPDIR", tmpFile) + t.Setenv("METIS_ROOTFS_TMP_DIR", tmpFile) if err := writeExt4Files(src, []inject.FileSpec{{Path: "etc/node.json", Content: []byte("{}"), Mode: 0o600, RootFS: true}}); err == nil { t.Fatal("expected writeExt4Files temp-dir failure") }