feat(firecracker): add migratable read-only virtio-fs

.dockerignore

@@ -15,6 +15,7 @@ output/*
 !output/checkpoint-restore
 !output/firecracker-agent
 !output/firecracker
+!output/virtiofsd
 !output/firecracker-vmlinux
 !output/firecracker-initrd.img
 !output/kata/

AGENTS.md

@@ -85,16 +85,10 @@ stays synchronized with the implementation.
 
 # Firecracker Storage Contract
 
-The Firecracker adapter accepts only local or image-provider-backed regular
-EROFS files for its root filesystem and filesystem image mounts. An operator
-may opt into OCI/Nydus rootfs materialization with
-`plugin.runtime.firecracker.oci_rootfs_enabled`. Keep derived EROFS files
-content-addressed and inside storage owned by the source image manager: OCI
-artifacts follow final-chain GC, while Nydus artifacts follow daemon/bootstrap
-GC. Do not create an independent tag-keyed cache or artifact reference count.
-OCI image mounts remain unsupported by Firecracker.
-
-Per-sandbox Firecracker storage is limited to the private ext4 writable layer
-and runtime state. Bounded read-only regular-file injection is a separate
-startup-metadata mechanism used for files such as `resolv.conf`; it must not
-grow into general directory or writable host sharing.
+The Firecracker adapter uses local or image-provider-backed regular EROFS files by default. An operator may instead enable the migration-capable, read-only virtio-fs path with `plugin.runtime.firecracker.virtiofs_enabled`; that path accepts directory root filesystems and explicitly read-only host directory mounts and exports them through one sandbox-scoped virtiofsd. OCI and Nydus root filesystems require virtio-fs and are consumed directly from the directory mounted by the image manager; never eagerly materialize those directories as EROFS. OCI image mounts remain unsupported by Firecracker.
+
+Per-sandbox Firecracker storage is limited to the private ext4 writable layer,
+virtio-fs staging and restored live-memory files, and runtime state. Every
+virtio-fs export must remain read-only; never extend this path to writable host
+sharing. Bounded read-only regular-file injection remains a separate
+startup-metadata mechanism for files such as `resolv.conf`.

README.md

@@ -161,10 +161,7 @@ tools/               pinned protobuf code-generation image
 
 ## Known limitations
 
-- Kata Containers and Firecracker require a usable `/dev/kvm`; nodes without
-  KVM continue to support gVisor. Firecracker additionally requires a compatible
-  guest kernel/initrd, an EROFS root image, and the ext4 image tool. Nodes that
-  enable OCI/Nydus rootfs materialization also require `mkfs.erofs`.
+- Kata Containers and Firecracker require a usable `/dev/kvm`; nodes without KVM continue to support gVisor. Firecracker additionally requires a compatible guest kernel/initrd and the ext4 image tool. Its root filesystem may be an immutable EROFS image or a directory exported through virtio-fs; directory-backed OCI/Nydus roots require `virtiofs_enabled` and a compatible virtiofsd.
 - NVIDIA GPU sandboxes require runsc, a directory/lisafs-backed rootfs,
   `nvidia-container-cli`, accessible NVIDIA devices and userspace driver
   libraries, and a host driver supported by the pinned runsc nvproxy. Kata,

cmd/firecracker-agent/main.go

@@ -45,6 +45,7 @@ const (
 	containerLower     = "/container/lower"
 	containerOverlay   = "/container/overlay"
 	containerNative    = "/container/overlay/native"
+	containerShared    = "/container/shared"
 	sandboxInitMode    = "sandbox-init"
 	sandboxConfigFD    = 3
 	sandboxStatusFD    = 4
@@ -296,8 +297,8 @@ func configure(request firecrackerproto.ConfigureRequest) error {
 	if state.configured {
 		return errors.New("sandbox is already configured")
 	}
-	if request.RootDevice == "" || request.OverlayDevice == "" {
-		return errors.New("root and overlay block devices are required")
+	if request.OverlayDevice == "" {
+		return errors.New("overlay block device is required")
 	}
 	if len(request.Process.Args) == 0 {
 		return errors.New("sandbox command is empty")
@@ -306,21 +307,49 @@ func configure(request firecrackerproto.ConfigureRequest) error {
 		containerRoot,
 		containerLower,
 		containerOverlay,
+		containerShared,
 		filepath.Join(containerOverlay, "upper"),
 		filepath.Join(containerOverlay, "work"),
 	} {
 		if err := os.MkdirAll(path, 0755); err != nil {
 			return err
 		}
 	}
-	if err := unix.Mount(
-		request.RootDevice,
-		containerLower,
-		"erofs",
-		unix.MS_RDONLY|unix.MS_NODEV,
-		"",
-	); err != nil {
-		return fmt.Errorf("mount root EROFS %s: %w", request.RootDevice, err)
+	rootFSType := request.RootFSType
+	if rootFSType == "" {
+		rootFSType = "erofs"
+	}
+	if request.VirtioFSTag != "" {
+		if err := mountSharedVirtioFS(request.VirtioFSTag); err != nil {
+			return err
+		}
+	}
+	lowerDir := containerLower
+	switch rootFSType {
+	case "erofs":
+		if request.RootDevice == "" {
+			return errors.New("EROFS root block device is required")
+		}
+		if err := unix.Mount(
+			request.RootDevice,
+			containerLower,
+			"erofs",
+			unix.MS_RDONLY|unix.MS_NODEV,
+			"",
+		); err != nil {
+			return fmt.Errorf("mount root EROFS %s: %w", request.RootDevice, err)
+		}
+	case "virtiofs":
+		if request.VirtioFSTag == "" {
+			return errors.New("virtio-fs root requires a mount tag")
+		}
+		var err error
+		lowerDir, err = sharedVirtioFSDirectory(request.RootSource)
+		if err != nil {
+			return fmt.Errorf("resolve virtio-fs root %q: %w", request.RootSource, err)
+		}
+	default:
+		return fmt.Errorf("unsupported root filesystem %q", rootFSType)
 	}
 	if err := unix.Mount(
 		request.OverlayDevice,
@@ -341,7 +370,7 @@ func configure(request firecrackerproto.ConfigureRequest) error {
 	}
 	overlayData := fmt.Sprintf(
 		"lowerdir=%s,upperdir=%s,workdir=%s",
-		containerLower,
+		lowerDir,
 		upper,
 		work,
 	)
@@ -895,6 +924,8 @@ func mountGuestFilesystem(mount firecrackerproto.MountSpec) error {
 	switch mount.FSType {
 	case "erofs":
 		return mountGuestEROFS(mount)
+	case "virtiofs":
+		return mountGuestVirtioFS(mount)
 	case "tmpfs":
 		return mountGuestTmpfs(mount)
 	default:
@@ -948,6 +979,91 @@ func mountNativeWritableUnder(
 	return nil
 }
 
+func mountSharedVirtioFS(tag string) error {
+	if tag == "" {
+		return errors.New("virtio-fs mount tag is empty")
+	}
+	if err := unix.Mount(
+		tag,
+		containerShared,
+		"virtiofs",
+		unix.MS_RDONLY|unix.MS_NODEV,
+		"",
+	); err != nil {
+		return fmt.Errorf("mount shared virtio-fs %s: %w", tag, err)
+	}
+	return nil
+}
+
+func mountGuestVirtioFS(mount firecrackerproto.MountSpec) error {
+	if mount.Source == "" {
+		return errors.New("virtio-fs guest mount source is empty")
+	}
+	source, err := sharedVirtioFSDirectory(mount.Source)
+	if err != nil {
+		return err
+	}
+	target, err := ensureContainerDirectory(mount.Target)
+	if err != nil {
+		return err
+	}
+	if err := unix.Mount(source, target, "", unix.MS_BIND|unix.MS_REC, ""); err != nil {
+		return fmt.Errorf(
+			"bind virtio-fs source %s at %s: %w",
+			mount.Source,
+			mount.Target,
+			err,
+		)
+	}
+	flags := uintptr(unix.MS_BIND | unix.MS_REMOUNT | unix.MS_RDONLY | unix.MS_NODEV)
+	for _, option := range mount.Options {
+		switch option {
+		case "ro":
+		case "nodev":
+			flags |= unix.MS_NODEV
+		case "noexec":
+			flags |= unix.MS_NOEXEC
+		case "nosuid":
+			flags |= unix.MS_NOSUID
+		default:
+			return fmt.Errorf("unsupported virtio-fs mount option %q", option)
+		}
+	}
+	if err := unix.Mount("", target, "", flags, ""); err != nil {
+		return fmt.Errorf("remount virtio-fs target %s read-only: %w", mount.Target, err)
+	}
+	return nil
+}
+
+func sharedVirtioFSDirectory(relative string) (string, error) {
+	return sharedVirtioFSDirectoryUnder(containerShared, relative)
+}
+
+func sharedVirtioFSDirectoryUnder(root, relative string) (string, error) {
+	if relative == "" || filepath.IsAbs(relative) {
+		return "", fmt.Errorf("virtio-fs source %q is not a relative path", relative)
+	}
+	clean := filepath.Clean(relative)
+	if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") {
+		return "", fmt.Errorf("virtio-fs source %q escapes the shared root", relative)
+	}
+	current := root
+	for _, component := range strings.Split(clean, string(filepath.Separator)) {
+		current = filepath.Join(current, component)
+		info, err := os.Lstat(current)
+		if err != nil {
+			return "", err
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return "", fmt.Errorf("virtio-fs source %q traverses symlink %s", relative, current)
+		}
+		if !info.IsDir() {
+			return "", fmt.Errorf("virtio-fs source %q contains non-directory %s", relative, current)
+		}
+	}
+	return current, nil
+}
+
 func mountGuestEROFS(mount firecrackerproto.MountSpec) error {
 	if mount.Device == "" {
 		return errors.New("EROFS guest mount device is empty")

cmd/firecracker-agent/main_test.go

@@ -390,6 +390,42 @@ func TestFirecrackerTmpfsParameters(t *testing.T) {
 	}
 }
 
+func TestSharedVirtioFSDirectory(t *testing.T) {
+	root := t.TempDir()
+	nested := filepath.Join(root, "mounts", "0001")
+	if err := os.MkdirAll(nested, 0755); err != nil {
+		t.Fatal(err)
+	}
+	resolved, err := sharedVirtioFSDirectoryUnder(root, "mounts/0001")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resolved != nested {
+		t.Fatalf("resolved virtio-fs directory = %q, want %q", resolved, nested)
+	}
+	for _, invalid := range []string{"", ".", "..", "../escape", "/absolute"} {
+		if _, err := sharedVirtioFSDirectoryUnder(root, invalid); err == nil {
+			t.Fatalf("accepted virtio-fs source %q", invalid)
+		}
+	}
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := sharedVirtioFSDirectoryUnder(root, "escape/subdir"); err == nil ||
+		!strings.Contains(err.Error(), "traverses symlink") {
+		t.Fatalf("symlink virtio-fs source error = %v", err)
+	}
+	file := filepath.Join(root, "file")
+	if err := os.WriteFile(file, []byte("data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := sharedVirtioFSDirectoryUnder(root, "file"); err == nil ||
+		!strings.Contains(err.Error(), "non-directory") {
+		t.Fatalf("file virtio-fs source error = %v", err)
+	}
+}
+
 func TestCheckpointHandoff(t *testing.T) {
 	root := t.TempDir()
 	environment := []string{

config/config.go

@@ -175,12 +175,13 @@ type FirecrackerConfig struct {
 	// generation as a Full snapshot. "incremental" enables the three-tier
 	// chain against a VMM that supports Incremental and SoftDirty snapshots.
 	CheckpointMode string `toml:"checkpoint_mode" json:"checkpointMode"`
-	// OCIRootfsEnabled permits an OCI image rootfs to be materialized as a
-	// local EROFS image before the Firecracker VM starts. It is opt-in because
-	// conversion eagerly reads the complete merged image.
-	OCIRootfsEnabled bool `toml:"oci_rootfs_enabled" json:"ociRootfsEnabled"`
-	// MkfsEROFSPath selects the mkfs.erofs executable used for materialization.
-	MkfsEROFSPath string `toml:"mkfs_erofs_path" json:"mkfsEROFSPath"`
+	// VirtioFSEnabled allows directory-backed root filesystems and read-only
+	// directory mounts to be exported through one sandbox-scoped virtiofsd.
+	// It requires the migration-capable Firecracker build and guest kernel.
+	VirtioFSEnabled bool `toml:"virtiofs_enabled" json:"virtiofsEnabled"`
+	// VirtioFSDPath selects the upstream virtiofsd executable. The runtime
+	// requires vhost-user DEVICE_STATE and LOG_SHMFD support.
+	VirtioFSDPath string `toml:"virtiofsd_path" json:"virtiofsdPath"`
 }
 
 type ResourceConfig struct {
@@ -320,6 +321,7 @@ func DefaultConfig() Config {
 					DefaultVCPUCount:        DefaultFirecrackerVCPUs,
 					DefaultMemoryMiB:        DefaultFirecrackerMemoryMiB,
 					DefaultOverlaySizeBytes: DefaultFirecrackerOverlayBytes,
+					VirtioFSDPath:           DefaultFirecrackerVirtioFSD,
 				},
 				ImageLibDir:              DefaultImageLibDir,
 				FilestoreDir:             DefaultFilestoreDir,

config/defaults.go

@@ -59,7 +59,7 @@ const (
 	DefaultFirecrackerVCPUs        = uint32(1)
 	DefaultFirecrackerMemoryMiB    = uint32(512)
 	DefaultFirecrackerOverlayBytes = uint64(10 << 30)
-	DefaultFirecrackerMkfsEROFS    = "mkfs.erofs"
+	DefaultFirecrackerVirtioFSD    = "/usr/local/bin/virtiofsd"
 
 	DefaultKataDANConfigDir = "/run/kata-containers/dans"
 )

config/runtime_files_test.go

@@ -39,7 +39,8 @@ func TestDefaultFirecrackerPaths(t *testing.T) {
 		fc.KVMDevice != DefaultKVMDevice ||
 		fc.DefaultVCPUCount != DefaultFirecrackerVCPUs ||
 		fc.DefaultMemoryMiB != DefaultFirecrackerMemoryMiB ||
-		fc.DefaultOverlaySizeBytes != DefaultFirecrackerOverlayBytes {
+		fc.DefaultOverlaySizeBytes != DefaultFirecrackerOverlayBytes ||
+		fc.VirtioFSDPath != DefaultFirecrackerVirtioFSD {
 		t.Fatalf("unexpected firecracker defaults: %+v", fc)
 	}
 }

configs/sandboxd.toml

@@ -98,11 +98,11 @@ runsc = ""
 # behavior where every generation is a Full snapshot; "incremental" enables
 # the three-tier incremental chain and requires a fork VMM that supports it.
 #checkpoint_mode = "full"
-# Opt-in support for materializing an OCI or Nydus image-manager rootfs mount
-# as an immutable EROFS file before starting a Firecracker VM. The generated
-# file follows the source image cache lifecycle.
-#oci_rootfs_enabled = false
-#mkfs_erofs_path = "/usr/bin/mkfs.erofs"
+# Opt-in support for directory-backed root filesystems and explicitly read-only
+# host directory mounts through one sandbox-scoped virtiofsd. This requires
+# the migration-capable Firecracker build and a guest kernel with virtio-fs.
+#virtiofs_enabled = false
+#virtiofsd_path = "/usr/local/bin/virtiofsd"
 
 [plugin.runtime.runtime_binary]
 runsc = "/usr/local/bin/runsc"