runtime(metis): move remote workspace onto usb scratch

cmd/metis/remote_cmd.go

@@ -23,7 +23,7 @@ import (
 func remoteDevicesCmd(args []string) {
 	fs := flag.NewFlagSet("remote-devices", flag.ExitOnError)
 	maxBytes := fs.Int64("max-device-bytes", 300000000000, "max real removable device size")
-	hostTmpDir := fs.String("host-tmp-dir", "/tmp/metis-flash-test", "host tmp dir for test writes")
+	hostTmpDir := fs.String("host-tmp-dir", "/var/tmp/metis-flash-test", "host tmp dir for test writes")
 	fs.Parse(args)
 
 	devices, err := localFlashDevices(*maxBytes, *hostTmpDir)
@@ -166,7 +166,7 @@ func remoteFlashCmd(args []string) {
 	harborRegistry := fs.String("harbor-registry", getenvOr("METIS_HARBOR_REGISTRY", "registry.bstein.dev"), "harbor registry host")
 	harborUsername := fs.String("harbor-username", getenvOr("METIS_HARBOR_USERNAME", ""), "harbor username")
 	harborPassword := fs.String("harbor-password", getenvOr("METIS_HARBOR_PASSWORD", ""), "harbor password")
-	hostTmpDir := fs.String("host-tmp-dir", "/host-tmp/metis-flash-test", "mounted host tmp dir for test writes")
+	hostTmpDir := fs.String("host-tmp-dir", "/host-tmp", "mounted host tmp dir for test writes")
 	fs.Parse(args)
 	if *node == "" || *device == "" || *artifactRef == "" {
 		fatalf("--node, --device, and --artifact-ref are required")
@@ -349,13 +349,17 @@ func localFlashDevices(maxBytes int64, hostTmpDir string) ([]service.Device, err
 			SizeBytes: size,
 		})
 	}
+	displayPath := humanHostPath(hostTmpDir)
+	if strings.TrimSpace(displayPath) == "" {
+		displayPath = "/var/tmp/metis-flash-test"
+	}
 	devices = append(devices, service.Device{
 		Name:      "host-tmp",
-		Path:      "hosttmp:///tmp",
+		Path:      "hosttmp://" + displayPath,
-		Model:     "Host /tmp",
+		Model:     "Host scratch",
 		Transport: "test",
 		Type:      "file",
-		Note:      fmt.Sprintf("Test-only host write target under %s", humanHostPath(hostTmpDir)),
+		Note:      fmt.Sprintf("Test-only host write target under %s", displayPath),
 		Removable: false,
 		Hotplug:   false,
 		SizeBytes: 1,

pkg/service/cluster.go

@@ -22,6 +22,8 @@ type clusterNode struct {
 	Worker                 bool
 	ControlPlane           bool
 	Unschedulable          bool
+	USBScratchStatus       string
+	USBScratchManagedPaths string
 }
 
 type podState struct {
@@ -173,6 +175,7 @@ func clusterNodes() []clusterNode {
 			Metadata struct {
 				Name        string            `json:"name"`
 				Labels      map[string]string `json:"labels"`
+				Annotations map[string]string `json:"annotations"`
 			} `json:"metadata"`
 			Spec struct {
 				Unschedulable bool `json:"unschedulable"`
@@ -185,6 +188,7 @@ func clusterNodes() []clusterNode {
 	nodes := make([]clusterNode, 0, len(payload.Items))
 	for _, item := range payload.Items {
 		labels := item.Metadata.Labels
+		annotations := item.Metadata.Annotations
 		nodes = append(nodes, clusterNode{
 			Name:                   strings.TrimSpace(item.Metadata.Name),
 			Arch:                   strings.TrimSpace(labels["kubernetes.io/arch"]),
@@ -192,6 +196,8 @@ func clusterNodes() []clusterNode {
 			Worker:                 labels["node-role.kubernetes.io/worker"] == "true",
 			ControlPlane:           labels["node-role.kubernetes.io/control-plane"] != "" || labels["node-role.kubernetes.io/master"] != "",
 			Unschedulable:          item.Spec.Unschedulable,
+			USBScratchStatus:       strings.TrimSpace(annotations["maintenance.bstein.dev/usb-scratch-status"]),
+			USBScratchManagedPaths: strings.TrimSpace(annotations["maintenance.bstein.dev/usb-scratch-managed-paths"]),
 		})
 	}
 	sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name })

pkg/service/remote.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	hostTmpDevicePath      = "hosttmp:///tmp"
+	hostTmpDevicePath      = "hosttmp:///var/tmp/metis-flash-test"
 	vaultRoleMaintenance   = "maintenance"
 	vaultRuntimeSecretPath = "kv/data/atlas/maintenance/metis-runtime"
 	vaultHarborSecretPath  = "kv/data/atlas/harbor/harbor-core"

pkg/service/remote_error_test.go

@@ -337,11 +337,11 @@ func remoteTestApp(t *testing.T, harbor *httptest.Server) *App {
 func remoteWorkflowKubeServer(t *testing.T, opts remoteKubeOptions) *httptest.Server {
 	t.Helper()
 	devicePhase := defaultString(opts.devicePhase, "Succeeded")
-	deviceMessage := defaultString(opts.deviceMessage, `{"devices":[{"name":"sdz","path":"/dev/sdz","model":"Micro SD","transport":"usb","type":"disk","removable":true,"hotplug":true,"size_bytes":32000000000},{"name":"tmp","path":"hosttmp:///tmp","model":"Host /tmp","transport":"test","type":"file","note":"Test-only host write target under /tmp","size_bytes":1}]}`)
+	deviceMessage := defaultString(opts.deviceMessage, `{"devices":[{"name":"sdz","path":"/dev/sdz","model":"Micro SD","transport":"usb","type":"disk","removable":true,"hotplug":true,"size_bytes":32000000000},{"name":"tmp","path":"hosttmp:///var/tmp/metis-flash-test","model":"Host scratch","transport":"test","type":"file","note":"Test-only host write target under /var/tmp/metis-flash-test","size_bytes":1}]}`)
 	buildPhase := defaultString(opts.buildPhase, "Succeeded")
 	buildMessage := defaultString(opts.buildMessage, `{"local_path":"/workspace/build/titan-15.img.xz","compressed":true,"size_bytes":1234,"build_tag":"build-1"}`)
 	flashPhase := defaultString(opts.flashPhase, "Succeeded")
-	flashMessage := defaultString(opts.flashMessage, `{"dest_path":"/tmp/metis-flash-test/titan-15.img"}`)
+	flashMessage := defaultString(opts.flashMessage, `{"dest_path":"/var/tmp/metis-flash-test/titan-15.img"}`)
 	nodes := opts.nodes
 	if nodes == nil {
 		nodes = []map[string]any{

pkg/service/remote_helpers.go

@@ -42,7 +42,7 @@ func flashStageHeartbeat(host, artifact string, elapsed time.Duration) (float64,
 func prettyDeviceTarget(path string) string {
 	switch {
 	case strings.HasPrefix(path, "hosttmp://"):
-		return "/tmp"
+		return strings.TrimPrefix(path, "hosttmp://")
 	case strings.TrimSpace(path) == "":
 		return "the selected target"
 	default:
@@ -50,6 +50,42 @@ func prettyDeviceTarget(path string) string {
 	}
 }
 
+func hostTmpHostPath(path string) string {
+	clean := filepath.Clean(strings.TrimSpace(path))
+	if clean == "" || clean == "." || clean == "/" {
+		return "/var/tmp/metis-flash-test"
+	}
+	return clean
+}
+
+func remoteWorkspaceHostPath(root, podName string) string {
+	cleanRoot := filepath.Clean(strings.TrimSpace(root))
+	if cleanRoot == "" || cleanRoot == "." || cleanRoot == "/" {
+		cleanRoot = "/var/tmp/metis-workspace"
+	}
+	if strings.TrimSpace(podName) == "" {
+		return cleanRoot
+	}
+	return filepath.Join(cleanRoot, podName)
+}
+
+func managedPathsContain(raw, want string) bool {
+	want = strings.TrimSpace(want)
+	if want == "" {
+		return false
+	}
+	for _, path := range strings.Split(raw, "_") {
+		if strings.TrimSpace(path) == want {
+			return true
+		}
+	}
+	return false
+}
+
+func usbScratchReadyForWorkspace(node clusterNode) bool {
+	return node.USBScratchStatus == "ok" && managedPathsContain(node.USBScratchManagedPaths, "/var/tmp")
+}
+
 func ramp(value, start, end, min, max float64) float64 {
 	if end <= start {
 		return max
@@ -107,6 +143,13 @@ func (a *App) selectBuilderHost(arch, flashHost string) (clusterNode, error) {
 			if node.Hardware == "rpi5" {
 				score += 30
 			}
+			if usbScratchReadyForWorkspace(node) {
+				score += 120
+			} else if node.USBScratchStatus == "error" {
+				score -= 200
+			} else {
+				score -= 80
+			}
 			if _, storage := storageNodes[node.Name]; storage {
 				score -= 50
 			}
@@ -164,14 +207,13 @@ func (a *App) remoteDevicePodSpec(name, host, image string) map[string]any {
 					"command": []string{
 						"metis", "remote-devices",
 						"--max-device-bytes", fmt.Sprintf("%d", a.settings.MaxDeviceBytes),
-						"--host-tmp-dir", mountedHostTmpDir(a.settings.HostTmpDir),
+						"--host-tmp-dir", hostTmpHostPath(a.settings.HostTmpDir),
 					},
 					"securityContext": map[string]any{"privileged": true, "runAsUser": 0},
 					"volumeMounts": []map[string]any{
 						{"name": "host-dev", "mountPath": "/dev"},
 						{"name": "host-sys", "mountPath": "/sys", "readOnly": true},
 						{"name": "host-udev", "mountPath": "/run/udev", "readOnly": true},
-						{"name": "host-tmp", "mountPath": "/host-tmp"},
 					},
 				},
 			},
@@ -180,13 +222,13 @@ func (a *App) remoteDevicePodSpec(name, host, image string) map[string]any {
 				{"name": "host-dev", "hostPath": map[string]any{"path": "/dev"}},
 				{"name": "host-sys", "hostPath": map[string]any{"path": "/sys"}},
 				{"name": "host-udev", "hostPath": map[string]any{"path": "/run/udev"}},
-				{"name": "host-tmp", "hostPath": map[string]any{"path": "/tmp"}},
 			},
 		},
 	}
 }
 
 func (a *App) remoteBuildPodSpec(name, host, image, node, artifactRef, buildTag string) map[string]any {
+	workspaceHostPath := remoteWorkspaceHostPath(a.settings.RemoteWorkspaceDir, name)
 	return map[string]any{
 		"apiVersion": "v1",
 		"kind":       "Pod",
@@ -231,13 +273,15 @@ func (a *App) remoteBuildPodSpec(name, host, image, node, artifactRef, buildTag
 			},
 			"imagePullSecrets": []map[string]string{{"name": "harbor-regcred"}},
 			"volumes": []map[string]any{
-				{"name": "workspace", "emptyDir": map[string]any{}},
+				{"name": "workspace", "hostPath": map[string]any{"path": workspaceHostPath, "type": "DirectoryOrCreate"}},
 			},
 		},
 	}
 }
 
 func (a *App) remoteFlashPodSpec(name, host, image, node, device, artifactRef string) map[string]any {
+	workspaceHostPath := remoteWorkspaceHostPath(a.settings.RemoteWorkspaceDir, name)
+	hostTmpPath := hostTmpHostPath(a.settings.HostTmpDir)
 	return map[string]any{
 		"apiVersion": "v1",
 		"kind":       "Pod",
@@ -286,11 +330,11 @@ func (a *App) remoteFlashPodSpec(name, host, image, node, device, artifactRef st
 			},
 			"imagePullSecrets": []map[string]string{{"name": "harbor-regcred"}},
 			"volumes": []map[string]any{
-				{"name": "workspace", "emptyDir": map[string]any{}},
+				{"name": "workspace", "hostPath": map[string]any{"path": workspaceHostPath, "type": "DirectoryOrCreate"}},
 				{"name": "host-dev", "hostPath": map[string]any{"path": "/dev"}},
 				{"name": "host-sys", "hostPath": map[string]any{"path": "/sys"}},
 				{"name": "host-udev", "hostPath": map[string]any{"path": "/run/udev"}},
-				{"name": "host-tmp", "hostPath": map[string]any{"path": "/tmp"}},
+				{"name": "host-tmp", "hostPath": map[string]any{"path": hostTmpPath, "type": "DirectoryOrCreate"}},
 			},
 		},
 	}
@@ -311,15 +355,7 @@ func inventoryNodeArch(spec *inventory.NodeSpec, class *inventory.NodeClass) str
 }
 
 func mountedHostTmpDir(path string) string {
-	path = strings.TrimSpace(path)
-	switch {
-	case path == "", path == "/tmp":
 	return "/host-tmp"
-	case strings.HasPrefix(path, "/tmp/"):
-		return filepath.Join("/host-tmp", strings.TrimPrefix(path, "/tmp/"))
-	default:
-		return filepath.Join("/host-tmp", strings.TrimPrefix(path, "/"))
-	}
 }
 
 func vaultRuntimeAnnotations(includeSSHKeys bool) map[string]string {

pkg/service/remote_helpers_test.go

@@ -15,7 +15,7 @@ func TestRemoteHelperBranches(t *testing.T) {
 	if got := prettyDeviceTarget(""); got != "the selected target" {
 		t.Fatalf("prettyDeviceTarget empty = %q", got)
 	}
-	if got := prettyDeviceTarget("hosttmp:///tmp"); got != "/tmp" {
+	if got := prettyDeviceTarget("hosttmp:///var/tmp/metis-flash-test"); got != "/var/tmp/metis-flash-test" {
 		t.Fatalf("prettyDeviceTarget hosttmp = %q", got)
 	}
 	if got := ramp(0, 10, 20, 1, 2); got != 1 {
@@ -27,10 +27,10 @@ func TestRemoteHelperBranches(t *testing.T) {
 	if got := ramp(20, 10, 20, 1, 2); got != 2 {
 		t.Fatalf("ramp after end = %v", got)
 	}
-	if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp/metis-flash-test" {
+	if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp" {
 		t.Fatalf("mountedHostTmpDir = %q", got)
 	}
-	if got := mountedHostTmpDir("/var/tmp/metis-flash-test"); got != "/host-tmp/var/tmp/metis-flash-test" {
+	if got := mountedHostTmpDir("/var/tmp/metis-flash-test"); got != "/host-tmp" {
 		t.Fatalf("mountedHostTmpDir non-tmp = %q", got)
 	}
 	if got := shellQuote(""); got != "''" {
@@ -247,6 +247,84 @@ func TestSelectBuilderHostScoresStorageAndAMD64(t *testing.T) {
 	}
 }
 
+func TestRemoteWorkspaceAndHostTmpPathsPreferUsbScratchRoots(t *testing.T) {
+	app := newTestApp(t)
+	app.settings.RemoteWorkspaceDir = "/var/tmp/metis-workspace"
+	app.settings.HostTmpDir = "/var/tmp/metis-flash-test"
+
+	buildSpec := app.remoteBuildPodSpec("metis-build-123", "titan-04", "runner:arm64", "titan-10", "registry.example/metis/titan-10", "build-1")
+	buildVolumes := buildSpec["spec"].(map[string]any)["volumes"].([]map[string]any)
+	workspaceVolume := buildVolumes[0]["hostPath"].(map[string]any)
+	if got := workspaceVolume["path"]; got != "/var/tmp/metis-workspace/metis-build-123" {
+		t.Fatalf("build workspace hostPath = %v", got)
+	}
+
+	flashSpec := app.remoteFlashPodSpec("metis-flash-123", "titan-04", "runner:arm64", "titan-10", hostTmpDevicePath, "registry.example/metis/titan-10")
+	flashVolumes := flashSpec["spec"].(map[string]any)["volumes"].([]map[string]any)
+	flashWorkspace := flashVolumes[0]["hostPath"].(map[string]any)
+	if got := flashWorkspace["path"]; got != "/var/tmp/metis-workspace/metis-flash-123" {
+		t.Fatalf("flash workspace hostPath = %v", got)
+	}
+	hostTmp := flashVolumes[4]["hostPath"].(map[string]any)
+	if got := hostTmp["path"]; got != "/var/tmp/metis-flash-test" {
+		t.Fatalf("host tmp hostPath = %v", got)
+	}
+}
+
+func TestSelectBuilderHostPrefersUsbScratchReadyWorkers(t *testing.T) {
+	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/nodes":
+			_ = json.NewEncoder(w).Encode(map[string]any{
+				"items": []any{
+					map[string]any{
+						"metadata": map[string]any{
+							"name": "titan-04",
+							"labels": map[string]string{
+								"kubernetes.io/arch":             "arm64",
+								"hardware":                       "rpi5",
+								"node-role.kubernetes.io/worker": "true",
+							},
+							"annotations": map[string]string{
+								"maintenance.bstein.dev/usb-scratch-status":        "ok",
+								"maintenance.bstein.dev/usb-scratch-managed-paths": "/var/log/pods_/var/tmp",
+							},
+						},
+						"spec": map[string]any{"unschedulable": false},
+					},
+					map[string]any{
+						"metadata": map[string]any{
+							"name": "titan-05",
+							"labels": map[string]string{
+								"kubernetes.io/arch":             "arm64",
+								"hardware":                       "rpi5",
+								"node-role.kubernetes.io/worker": "true",
+							},
+						},
+						"spec": map[string]any{"unschedulable": false},
+					},
+				},
+			})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/maintenance/pods":
+			_ = json.NewEncoder(w).Encode(map[string]any{"items": []any{}})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer kube.Close()
+	installKubeFactory(t, kube)
+	app := newTestApp(t)
+	app.settings.Namespace = "maintenance"
+
+	node, err := app.selectBuilderHost("arm64", "")
+	if err != nil {
+		t.Fatalf("selectBuilderHost: %v", err)
+	}
+	if node.Name != "titan-04" {
+		t.Fatalf("expected titan-04 scratch-ready builder, got %s", node.Name)
+	}
+}
+
 func TestSelectBuilderHostTieBreaksByName(t *testing.T) {
 	kube := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch {

pkg/service/remote_test.go

@@ -6,8 +6,8 @@ import (
 )
 
 func TestMountedHostTmpDirMapsConfiguredTmpPathIntoMount(t *testing.T) {
-	if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp/metis-flash-test" {
+	if got := mountedHostTmpDir("/tmp/metis-flash-test"); got != "/host-tmp" {
-		t.Fatalf("expected /host-tmp/metis-flash-test, got %q", got)
+		t.Fatalf("expected /host-tmp, got %q", got)
 	}
 	if got := mountedHostTmpDir("/tmp"); got != "/host-tmp" {
 		t.Fatalf("expected /host-tmp, got %q", got)

pkg/service/settings.go

@@ -33,6 +33,7 @@ type Settings struct {
 	HarborUsername     string
 	HarborPassword     string
 	HostTmpDir         string
+	RemoteWorkspaceDir string
 	RemotePodTimeout   int64
 }
 
@@ -64,7 +65,8 @@ func FromEnv() Settings {
 		HarborAPIBase:      getenvDefault("METIS_HARBOR_API_BASE", "https://registry.bstein.dev/api/v2.0"),
 		HarborUsername:     getenvDefault("METIS_HARBOR_USERNAME", ""),
 		HarborPassword:     getenvDefault("METIS_HARBOR_PASSWORD", ""),
-		HostTmpDir:        getenvDefault("METIS_HOST_TMP_DIR", "/tmp/metis-flash-test"),
+		HostTmpDir:         getenvDefault("METIS_HOST_TMP_DIR", "/var/tmp/metis-flash-test"),
+		RemoteWorkspaceDir: getenvDefault("METIS_REMOTE_WORKSPACE_DIR", "/var/tmp/metis-workspace"),
 		RemotePodTimeout:   getenvInt64("METIS_REMOTE_POD_TIMEOUT_SEC", 1800),
 	}
 }

pkg/service/workflow_test.go

@@ -181,11 +181,11 @@ func fakeKubeServer(t *testing.T) *httptest.Server {
 			message := `{}`
 			switch {
 			case strings.Contains(podName, "devices"):
-				message = `{"devices":[{"name":"sdz","path":"/dev/sdz","model":"Micro SD","transport":"usb","type":"disk","removable":true,"hotplug":true,"size_bytes":32000000000},{"name":"tmp","path":"hosttmp:///tmp","model":"Host /tmp","transport":"test","type":"file","note":"Test-only host write target under /tmp","size_bytes":1}]}`
+				message = `{"devices":[{"name":"sdz","path":"/dev/sdz","model":"Micro SD","transport":"usb","type":"disk","removable":true,"hotplug":true,"size_bytes":32000000000},{"name":"tmp","path":"hosttmp:///var/tmp/metis-flash-test","model":"Host scratch","transport":"test","type":"file","note":"Test-only host write target under /var/tmp/metis-flash-test","size_bytes":1}]}`
 			case strings.Contains(podName, "build"):
 				message = `{"local_path":"/workspace/build/titan-15.img.xz","compressed":true,"size_bytes":1234,"build_tag":"build-1"}`
 			case strings.Contains(podName, "flash"):
-				message = `{"dest_path":"/tmp/metis-flash-test/titan-15.img"}`
+				message = `{"dest_path":"/var/tmp/metis-flash-test/titan-15.img"}`
 			}
 			_ = json.NewEncoder(w).Encode(map[string]any{
 				"metadata": map[string]any{"name": podName},