diff --git a/.dockerignore b/.dockerignore index 2be03b6..e09a611 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,7 @@ output/* !output/checkpoint-restore !output/firecracker-agent !output/firecracker +!output/virtiofsd !output/firecracker-vmlinux !output/firecracker-initrd.img !output/kata/ diff --git a/AGENTS.md b/AGENTS.md index 1879c2a..c18d8b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,16 +85,10 @@ stays synchronized with the implementation. # Firecracker Storage Contract -The Firecracker adapter accepts only local or image-provider-backed regular -EROFS files for its root filesystem and filesystem image mounts. An operator -may opt into OCI/Nydus rootfs materialization with -`plugin.runtime.firecracker.oci_rootfs_enabled`. Keep derived EROFS files -content-addressed and inside storage owned by the source image manager: OCI -artifacts follow final-chain GC, while Nydus artifacts follow daemon/bootstrap -GC. Do not create an independent tag-keyed cache or artifact reference count. -OCI image mounts remain unsupported by Firecracker. - -Per-sandbox Firecracker storage is limited to the private ext4 writable layer -and runtime state. Bounded read-only regular-file injection is a separate -startup-metadata mechanism used for files such as `resolv.conf`; it must not -grow into general directory or writable host sharing. +The Firecracker adapter uses local or image-provider-backed regular EROFS files by default. An operator may instead enable the migration-capable, read-only virtio-fs path with `plugin.runtime.firecracker.virtiofs_enabled`; that path accepts directory root filesystems and explicitly read-only host directory mounts and exports them through one sandbox-scoped virtiofsd. OCI and Nydus root filesystems require virtio-fs and are consumed directly from the directory mounted by the image manager; never eagerly materialize those directories as EROFS. OCI image mounts remain unsupported by Firecracker. + +Per-sandbox Firecracker storage is limited to the private ext4 writable layer, +virtio-fs staging and restored live-memory files, and runtime state. Every +virtio-fs export must remain read-only; never extend this path to writable host +sharing. Bounded read-only regular-file injection remains a separate +startup-metadata mechanism for files such as `resolv.conf`. diff --git a/README.md b/README.md index 3919e19..f22f9c2 100644 --- a/README.md +++ b/README.md @@ -161,10 +161,7 @@ tools/ pinned protobuf code-generation image ## Known limitations -- Kata Containers and Firecracker require a usable `/dev/kvm`; nodes without - KVM continue to support gVisor. Firecracker additionally requires a compatible - guest kernel/initrd, an EROFS root image, and the ext4 image tool. Nodes that - enable OCI/Nydus rootfs materialization also require `mkfs.erofs`. +- Kata Containers and Firecracker require a usable `/dev/kvm`; nodes without KVM continue to support gVisor. Firecracker additionally requires a compatible guest kernel/initrd and the ext4 image tool. Its root filesystem may be an immutable EROFS image or a directory exported through virtio-fs; directory-backed OCI/Nydus roots require `virtiofs_enabled` and a compatible virtiofsd. - NVIDIA GPU sandboxes require runsc, a directory/lisafs-backed rootfs, `nvidia-container-cli`, accessible NVIDIA devices and userspace driver libraries, and a host driver supported by the pinned runsc nvproxy. Kata, diff --git a/cmd/firecracker-agent/main.go b/cmd/firecracker-agent/main.go index c9e57cf..a3cb7c1 100644 --- a/cmd/firecracker-agent/main.go +++ b/cmd/firecracker-agent/main.go @@ -45,6 +45,7 @@ const ( containerLower = "/container/lower" containerOverlay = "/container/overlay" containerNative = "/container/overlay/native" + containerShared = "/container/shared" sandboxInitMode = "sandbox-init" sandboxConfigFD = 3 sandboxStatusFD = 4 @@ -296,8 +297,8 @@ func configure(request firecrackerproto.ConfigureRequest) error { if state.configured { return errors.New("sandbox is already configured") } - if request.RootDevice == "" || request.OverlayDevice == "" { - return errors.New("root and overlay block devices are required") + if request.OverlayDevice == "" { + return errors.New("overlay block device is required") } if len(request.Process.Args) == 0 { return errors.New("sandbox command is empty") @@ -306,6 +307,7 @@ func configure(request firecrackerproto.ConfigureRequest) error { containerRoot, containerLower, containerOverlay, + containerShared, filepath.Join(containerOverlay, "upper"), filepath.Join(containerOverlay, "work"), } { @@ -313,14 +315,41 @@ func configure(request firecrackerproto.ConfigureRequest) error { return err } } - if err := unix.Mount( - request.RootDevice, - containerLower, - "erofs", - unix.MS_RDONLY|unix.MS_NODEV, - "", - ); err != nil { - return fmt.Errorf("mount root EROFS %s: %w", request.RootDevice, err) + rootFSType := request.RootFSType + if rootFSType == "" { + rootFSType = "erofs" + } + if request.VirtioFSTag != "" { + if err := mountSharedVirtioFS(request.VirtioFSTag); err != nil { + return err + } + } + lowerDir := containerLower + switch rootFSType { + case "erofs": + if request.RootDevice == "" { + return errors.New("EROFS root block device is required") + } + if err := unix.Mount( + request.RootDevice, + containerLower, + "erofs", + unix.MS_RDONLY|unix.MS_NODEV, + "", + ); err != nil { + return fmt.Errorf("mount root EROFS %s: %w", request.RootDevice, err) + } + case "virtiofs": + if request.VirtioFSTag == "" { + return errors.New("virtio-fs root requires a mount tag") + } + var err error + lowerDir, err = sharedVirtioFSDirectory(request.RootSource) + if err != nil { + return fmt.Errorf("resolve virtio-fs root %q: %w", request.RootSource, err) + } + default: + return fmt.Errorf("unsupported root filesystem %q", rootFSType) } if err := unix.Mount( request.OverlayDevice, @@ -341,7 +370,7 @@ func configure(request firecrackerproto.ConfigureRequest) error { } overlayData := fmt.Sprintf( "lowerdir=%s,upperdir=%s,workdir=%s", - containerLower, + lowerDir, upper, work, ) @@ -895,6 +924,8 @@ func mountGuestFilesystem(mount firecrackerproto.MountSpec) error { switch mount.FSType { case "erofs": return mountGuestEROFS(mount) + case "virtiofs": + return mountGuestVirtioFS(mount) case "tmpfs": return mountGuestTmpfs(mount) default: @@ -948,6 +979,91 @@ func mountNativeWritableUnder( return nil } +func mountSharedVirtioFS(tag string) error { + if tag == "" { + return errors.New("virtio-fs mount tag is empty") + } + if err := unix.Mount( + tag, + containerShared, + "virtiofs", + unix.MS_RDONLY|unix.MS_NODEV, + "", + ); err != nil { + return fmt.Errorf("mount shared virtio-fs %s: %w", tag, err) + } + return nil +} + +func mountGuestVirtioFS(mount firecrackerproto.MountSpec) error { + if mount.Source == "" { + return errors.New("virtio-fs guest mount source is empty") + } + source, err := sharedVirtioFSDirectory(mount.Source) + if err != nil { + return err + } + target, err := ensureContainerDirectory(mount.Target) + if err != nil { + return err + } + if err := unix.Mount(source, target, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { + return fmt.Errorf( + "bind virtio-fs source %s at %s: %w", + mount.Source, + mount.Target, + err, + ) + } + flags := uintptr(unix.MS_BIND | unix.MS_REMOUNT | unix.MS_RDONLY | unix.MS_NODEV) + for _, option := range mount.Options { + switch option { + case "ro": + case "nodev": + flags |= unix.MS_NODEV + case "noexec": + flags |= unix.MS_NOEXEC + case "nosuid": + flags |= unix.MS_NOSUID + default: + return fmt.Errorf("unsupported virtio-fs mount option %q", option) + } + } + if err := unix.Mount("", target, "", flags, ""); err != nil { + return fmt.Errorf("remount virtio-fs target %s read-only: %w", mount.Target, err) + } + return nil +} + +func sharedVirtioFSDirectory(relative string) (string, error) { + return sharedVirtioFSDirectoryUnder(containerShared, relative) +} + +func sharedVirtioFSDirectoryUnder(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) { + return "", fmt.Errorf("virtio-fs source %q is not a relative path", relative) + } + clean := filepath.Clean(relative) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") { + return "", fmt.Errorf("virtio-fs source %q escapes the shared root", relative) + } + current := root + for _, component := range strings.Split(clean, string(filepath.Separator)) { + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if err != nil { + return "", err + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("virtio-fs source %q traverses symlink %s", relative, current) + } + if !info.IsDir() { + return "", fmt.Errorf("virtio-fs source %q contains non-directory %s", relative, current) + } + } + return current, nil +} + func mountGuestEROFS(mount firecrackerproto.MountSpec) error { if mount.Device == "" { return errors.New("EROFS guest mount device is empty") diff --git a/cmd/firecracker-agent/main_test.go b/cmd/firecracker-agent/main_test.go index 138480f..1e3f447 100644 --- a/cmd/firecracker-agent/main_test.go +++ b/cmd/firecracker-agent/main_test.go @@ -390,6 +390,42 @@ func TestFirecrackerTmpfsParameters(t *testing.T) { } } +func TestSharedVirtioFSDirectory(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "mounts", "0001") + if err := os.MkdirAll(nested, 0755); err != nil { + t.Fatal(err) + } + resolved, err := sharedVirtioFSDirectoryUnder(root, "mounts/0001") + if err != nil { + t.Fatal(err) + } + if resolved != nested { + t.Fatalf("resolved virtio-fs directory = %q, want %q", resolved, nested) + } + for _, invalid := range []string{"", ".", "..", "../escape", "/absolute"} { + if _, err := sharedVirtioFSDirectoryUnder(root, invalid); err == nil { + t.Fatalf("accepted virtio-fs source %q", invalid) + } + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + if _, err := sharedVirtioFSDirectoryUnder(root, "escape/subdir"); err == nil || + !strings.Contains(err.Error(), "traverses symlink") { + t.Fatalf("symlink virtio-fs source error = %v", err) + } + file := filepath.Join(root, "file") + if err := os.WriteFile(file, []byte("data"), 0644); err != nil { + t.Fatal(err) + } + if _, err := sharedVirtioFSDirectoryUnder(root, "file"); err == nil || + !strings.Contains(err.Error(), "non-directory") { + t.Fatalf("file virtio-fs source error = %v", err) + } +} + func TestCheckpointHandoff(t *testing.T) { root := t.TempDir() environment := []string{ diff --git a/config/config.go b/config/config.go index 72eedde..c8d5e08 100644 --- a/config/config.go +++ b/config/config.go @@ -175,12 +175,13 @@ type FirecrackerConfig struct { // generation as a Full snapshot. "incremental" enables the three-tier // chain against a VMM that supports Incremental and SoftDirty snapshots. CheckpointMode string `toml:"checkpoint_mode" json:"checkpointMode"` - // OCIRootfsEnabled permits an OCI image rootfs to be materialized as a - // local EROFS image before the Firecracker VM starts. It is opt-in because - // conversion eagerly reads the complete merged image. - OCIRootfsEnabled bool `toml:"oci_rootfs_enabled" json:"ociRootfsEnabled"` - // MkfsEROFSPath selects the mkfs.erofs executable used for materialization. - MkfsEROFSPath string `toml:"mkfs_erofs_path" json:"mkfsEROFSPath"` + // VirtioFSEnabled allows directory-backed root filesystems and read-only + // directory mounts to be exported through one sandbox-scoped virtiofsd. + // It requires the migration-capable Firecracker build and guest kernel. + VirtioFSEnabled bool `toml:"virtiofs_enabled" json:"virtiofsEnabled"` + // VirtioFSDPath selects the upstream virtiofsd executable. The runtime + // requires vhost-user DEVICE_STATE and LOG_SHMFD support. + VirtioFSDPath string `toml:"virtiofsd_path" json:"virtiofsdPath"` } type ResourceConfig struct { @@ -320,6 +321,7 @@ func DefaultConfig() Config { DefaultVCPUCount: DefaultFirecrackerVCPUs, DefaultMemoryMiB: DefaultFirecrackerMemoryMiB, DefaultOverlaySizeBytes: DefaultFirecrackerOverlayBytes, + VirtioFSDPath: DefaultFirecrackerVirtioFSD, }, ImageLibDir: DefaultImageLibDir, FilestoreDir: DefaultFilestoreDir, diff --git a/config/defaults.go b/config/defaults.go index f3b3e16..69ba97b 100644 --- a/config/defaults.go +++ b/config/defaults.go @@ -59,7 +59,7 @@ const ( DefaultFirecrackerVCPUs = uint32(1) DefaultFirecrackerMemoryMiB = uint32(512) DefaultFirecrackerOverlayBytes = uint64(10 << 30) - DefaultFirecrackerMkfsEROFS = "mkfs.erofs" + DefaultFirecrackerVirtioFSD = "/usr/local/bin/virtiofsd" DefaultKataDANConfigDir = "/run/kata-containers/dans" ) diff --git a/config/runtime_files_test.go b/config/runtime_files_test.go index 6e03fc8..8544577 100644 --- a/config/runtime_files_test.go +++ b/config/runtime_files_test.go @@ -39,7 +39,8 @@ func TestDefaultFirecrackerPaths(t *testing.T) { fc.KVMDevice != DefaultKVMDevice || fc.DefaultVCPUCount != DefaultFirecrackerVCPUs || fc.DefaultMemoryMiB != DefaultFirecrackerMemoryMiB || - fc.DefaultOverlaySizeBytes != DefaultFirecrackerOverlayBytes { + fc.DefaultOverlaySizeBytes != DefaultFirecrackerOverlayBytes || + fc.VirtioFSDPath != DefaultFirecrackerVirtioFSD { t.Fatalf("unexpected firecracker defaults: %+v", fc) } } diff --git a/configs/sandboxd.toml b/configs/sandboxd.toml index 09085fc..f45c42d 100644 --- a/configs/sandboxd.toml +++ b/configs/sandboxd.toml @@ -98,11 +98,11 @@ runsc = "" # behavior where every generation is a Full snapshot; "incremental" enables # the three-tier incremental chain and requires a fork VMM that supports it. #checkpoint_mode = "full" -# Opt-in support for materializing an OCI or Nydus image-manager rootfs mount -# as an immutable EROFS file before starting a Firecracker VM. The generated -# file follows the source image cache lifecycle. -#oci_rootfs_enabled = false -#mkfs_erofs_path = "/usr/bin/mkfs.erofs" +# Opt-in support for directory-backed root filesystems and explicitly read-only +# host directory mounts through one sandbox-scoped virtiofsd. This requires +# the migration-capable Firecracker build and a guest kernel with virtio-fs. +#virtiofs_enabled = false +#virtiofsd_path = "/usr/local/bin/virtiofsd" [plugin.runtime.runtime_binary] runsc = "/usr/local/bin/runsc" diff --git a/doc/checkpoint-restore.md b/doc/checkpoint-restore.md index 1856b7a..cc7353e 100644 --- a/doc/checkpoint-restore.md +++ b/doc/checkpoint-restore.md @@ -55,7 +55,8 @@ to the runtime that created them. The Firecracker runtime writes *uncompressed* checkpoint directories (layout version 2): `manifest.json` plus the `vmstate`, `memory`, and `overlay.ext4` -components. The manifest is written last as the logical commit marker: under +components. A VM with virtio-fs also carries `virtiofs.state`. The manifest is +written last as the logical commit marker: under normal same-boot operation, a directory that shows a manifest is complete; a directory without one is partial output that sandboxd cleans up. The memory file stays a plain file that Firecracker @@ -183,7 +184,8 @@ restart always marks the lineage lost for surviving sandboxes: the restart cannot tell which generation the surviving VMM is armed against, so the cheapest provably-safe recovery is one `Full` checkpoint per sandbox. -The manifest digests only the small VM state component. Hashing the memory file +The manifest digests the small VM state and optional virtiofsd state +components. Hashing the memory file or `overlay.ext4` is skipped because it costs seconds of CPU and page-cache reads per GiB and would dominate checkpoint latency. Their local integrity rests on reflink copy-on-write and Firecracker's own writes. Restores skip @@ -198,21 +200,34 @@ timestamp granularity goes undetected — the same granularity the nydus bootstrap cache accepts. The manifest also records a `compat` tuple — sha256 digests of the Firecracker -binary, guest kernel, and initrd, plus architecture and kernel arguments — +binary, guest kernel, and initrd, plus architecture and kernel arguments. A +virtio-fs checkpoint additionally records the virtiofsd digest. Values are computed once per sandboxd process. A restore compares the tuple against its own stack and refuses on a mismatch, naming the conflicting field. Manifests -without a tuple (artifacts from before the tuple existed) restore without -stack verification. +without a tuple (artifacts from before the tuple existed) restore without stack +verification. ### Storage layout for high-performance Firecracker checkpoints Firecracker memory and the writable block image are separate checkpoint components. Firecracker writes or patches `memory`; sandboxd snapshots the -live `overlay.ext4` into the artifact. Restore maps the artifact's `memory` -file in place and clones `overlay.ext4` into a new sandbox-owned writable -image. The artifact overlay must not be used as the restored VM's writable -image: checkpoint generations are immutable, the source may keep running, and -concurrent restores require independent writable layers. +live `overlay.ext4` into the artifact. A conventional restore maps the +artifact's `memory` file privately. A virtio-fs restore first reflink-clones it +to a sandbox-owned `memory.live` file (or copies it when reflink is unavailable) +and maps only that live file writable and shared, because virtiofsd must write +guest buffers directly. The committed checkpoint memory is never mapped +writable. Restore also clones `overlay.ext4` into a new sandbox-owned writable +image. Checkpoint components must not become live writable state: the source +may keep running, and concurrent restores require independent layers. + +For a virtio-fs checkpoint, Firecracker keeps `VHOST_F_LOG_ALL` armed for the +device lifetime. While the VM is paused it stops and drains both queues, +serializes virtiofsd into `virtiofs.state`, collects the shared vhost dirty +bitmap, and includes those guest-memory ranges in every snapshot flavor before +re-enabling the queues. Restore requires the same virtio-fs/non-virtio-fs +storage layout, starts a replacement virtiofsd over the newly prepared +read-only exports, loads its sidecar before enabling queues, and resumes the +guest only after the device and memory state agree. Firecracker native writable mounts do not add checkpoint components. Their directories reside in the same `overlay.ext4` as the root overlay's upper and @@ -351,11 +366,11 @@ If restore fails, sandboxd rolls back the partially created target. It does not modify the source or delete the checkpoint input. After `Start` succeeds, the target no longer depends on the checkpoint -directory — with one exception: restoring a Firecracker v2 directory keeps the -artifact's `memory` file mapped into the restored VM, so the caller must keep -the checkpoint directory intact until the restored sandbox exits. The next -checkpoint of the restored sandbox also diffs against that memory file -(the tier-2 base below). +directory — with one exception: a Firecracker v2 restore keeps the artifact's +`memory` file as its tier-2 incremental base. A conventional restore also maps +that file privately; a virtio-fs restore maps an independent shared live clone. +The caller must keep the checkpoint directory intact until the restored +sandbox exits or establishes a later complete checkpoint generation. ## Runtime support and compatibility diff --git a/doc/runtime.md b/doc/runtime.md index cf0a5ac..cd60520 100644 --- a/doc/runtime.md +++ b/doc/runtime.md @@ -9,10 +9,10 @@ binaries, boot artifacts, and host prerequisites pass validation. | Capability | runsc | runc | Kata Containers | Firecracker | | --- | --- | --- | --- | --- | | Kernel boundary | gVisor user-space kernel | Host Linux kernel | Dedicated guest kernel in a lightweight VM | Dedicated guest kernel in a microVM | -| Host requirements | Tested runsc binary; `/dev/kvm` when the KVM platform is selected | runc and runc-shim; writable cgroups, overlayfs, EROFS, and loop devices | Kata runtime and configuration with usable `/dev/kvm` | Firecracker, compatible kernel and initrd, `/dev/kvm`, `mkfs.ext4`, and optionally `mkfs.erofs` | +| Host requirements | Tested runsc binary; `/dev/kvm` when the KVM platform is selected | runc and runc-shim; writable cgroups, overlayfs, EROFS, and loop devices | Kata runtime and configuration with usable `/dev/kvm` | Firecracker, compatible kernel and initrd, `/dev/kvm`, `mkfs.ext4`, and virtiofsd when directory sharing is enabled | | Network lifecycle | Reusable TAP from the interface pool | New netns and veth per sandbox, deleted on release | Reusable TAP from the interface pool | Reusable TAP from the interface pool | -| Root filesystem | Directory or EROFS | Directory or EROFS with a host overlay | Directory or EROFS passed into the VM | Immutable EROFS drive or opt-in OCI/Nydus materialization, plus a private ext4 overlay | -| Read-only mounts | Bind, EROFS, and runtime-supported OCI mounts | Bind, EROFS, and OCI mounts | Bind, EROFS, and runtime-supported OCI mounts | EROFS drives and bounded regular-file injection | +| Root filesystem | Directory or EROFS | Directory or EROFS with a host overlay | Directory or EROFS passed into the VM | Immutable EROFS drive or opt-in virtio-fs directory, plus a private ext4 overlay | +| Read-only mounts | Bind, EROFS, and runtime-supported OCI mounts | Bind, EROFS, and OCI mounts | Bind, EROFS, and runtime-supported OCI mounts | EROFS drives, virtio-fs directories, and bounded regular-file injection | | Exec, interactive TTY, wait, stats, and recovery | Supported | Supported | Supported | Supported | | Network ACL and managed DNS | Supported | Not supported | Supported | Supported | | Published-port DNAT | Supported | Supported | Supported | Supported | @@ -44,9 +44,11 @@ uses `plugin.runtime.firecracker` and requires `plugin.runtime.filestore_dir`. An unavailable optional adapter is omitted while the other runtimes remain usable. -Firecracker uses the stock VMM API and expects KVM at `/dev/kvm`. Its kernel -must include virtio block, virtio net, vsock, EROFS, ext4, overlayfs, devtmpfs, -and the cgroup controllers needed by the guest. The initrd must contain the +Firecracker expects KVM at `/dev/kvm`. Its kernel must include virtio block, +virtio net, vsock, EROFS, ext4, overlayfs, devtmpfs, and the cgroup controllers +needed by the guest. The optional virtio-fs path additionally requires +`CONFIG_FUSE_FS=y` and `CONFIG_VIRTIO_FS=y`; DAX stays disabled because the VMM +does not expose a shared-memory window. The initrd must contain the matching sandboxd `firecracker-agent` as `/init`. Default artifact paths are `/opt/firecracker/vmlinux` and `/opt/firecracker/initrd.img`; the sample configuration shows all overrides. The default VM size is one vCPU and @@ -73,27 +75,19 @@ network ACLs. ## Firecracker storage model -Firecracker accepts only a regular file containing an EROFS superblock as its -root filesystem. The file may be local or exposed by an image provider such as -distill-fs, so object-storage range reads and lazy caching remain outside the -runtime adapter. - -Set `oci_rootfs_enabled = true` under `plugin.runtime.firecracker` to accept an -OCI image reference as the rootfs. sandboxd first mounts the image through its -existing OCI/Nydus image manager, then runs `mkfs.erofs --quiet --Enoinline_data` over the merged read-only directory. `mkfs_erofs_path` -selects the executable and defaults to `mkfs.erofs`. Conversion is eager and -therefore reads the complete image before the VM starts. - -The generated file does not use a separate tag-keyed cache. A regular OCI -image is keyed by its final chain ID and stored beside that chain, so the -existing chain TTL and disk-pressure GC remove it. A Nydus image is keyed by -the bootstrap digest and stored in the daemon directory, so daemon GC owns it. -This follows the same content-addressed ownership principle as the -[containerd EROFS snapshotter](https://github.com/containerd/containerd/blob/main/docs/snapshotters/erofs.md) -while retaining sandboxd's current image lifecycle. Creation uses a temporary -file and an atomic rename; sandboxd does not fsync the read-only derived -artifact. Firecracker OCI image mounts remain unsupported. +By default Firecracker accepts a regular file containing an EROFS superblock as +its root filesystem. The file may be local or exposed by an image provider +such as distill-fs, so object-storage range reads and lazy caching remain +outside the runtime adapter. + +Set `virtiofs_enabled = true` to use directory-backed root filesystems and explicitly read-only host-directory mounts, including OCI/Nydus rootfs directories resolved by the image manager. OCI image mounts remain unsupported. sandboxd creates one private staging tmpfs per sandbox, recursively bind-mounts each source below fixed relative paths, and starts one upstream virtiofsd selected by `virtiofsd_path` (default `/usr/local/bin/virtiofsd`). The daemon is always started with `--readonly`, namespace sandboxing, submount announcements disabled, inode file handles disabled, and `find-paths` migration mode. Disabling submount announcements makes the staging bind mounts ordinary virtio-fs directories in the guest, so they can serve as an overlayfs lower layer. The staging binds are also remounted read-only. The image manager keeps owning and garbage-collecting the source; Firecracker creates no independent image cache. OCI and Nydus rootfs directories require this mode and are never eagerly converted to EROFS. + +This mode requires the AKernel Firecracker build with the MMIO virtio-fs +frontend and vhost-user migration support, plus virtiofsd 1.14 or newer. The +frontend requires `MQ`, `REPLY_ACK`, `LOG_SHMFD`, `DEVICE_STATE`, and +`VHOST_F_LOG_ALL`; startup fails rather than silently disabling checkpoint +correctness when a backend lacks them. DAX and writable host sharing are not +supported. The sandbox's private ext4 overlay remains the only writable layer. Every sandbox gets a sparse ext4 image under `filestore_dir/.firecracker` and uses it as the overlay upper and work filesystem. For a read-only root, the @@ -127,11 +121,13 @@ requested. EROFS and `rofs` mounts must also name regular EROFS image files and are attached as read-only drives. Read-only regular files are injected into the guest, limited to 1 MiB per file and 4 MiB in total; this narrow path supports -managed files such as `resolv.conf` and does not provide directory sharing. At -most 24 drives, including root and overlay, may be attached. Directory roots -that were not explicitly materialized, directory binds, writable binds, host -device-provider OCI updates, NVIDIA devices, and nested KVM are rejected -instead of being silently weakened. +managed files such as `resolv.conf`. With virtio-fs disabled, directory roots +that were not explicitly materialized and directory binds are rejected. With +virtio-fs enabled, directory roots and explicitly read-only directory binds use +the single shared filesystem instead of block drives. At most 24 block drives, +including an EROFS root and the overlay, may be attached. Writable binds, host +device-provider OCI updates, NVIDIA devices, and nested KVM are always +rejected instead of being silently weakened. Private tmpfs mounts are supported with a bounded set of standard security, ownership, mode, inode, and size options. diff --git a/internal/firecrackerproto/protocol.go b/internal/firecrackerproto/protocol.go index 84619ed..f145367 100644 --- a/internal/firecrackerproto/protocol.go +++ b/internal/firecrackerproto/protocol.go @@ -27,7 +27,7 @@ import ( ) const ( - Version = uint16(1) + Version = uint16(2) AgentPort = uint32(52) maxMessageSize = 16 << 20 @@ -99,6 +99,7 @@ type NetworkSpec struct { type MountSpec struct { Device string `json:"device"` + Source string `json:"source,omitempty"` Target string `json:"target"` FSType string `json:"fs_type"` Options []string `json:"options,omitempty"` @@ -120,6 +121,9 @@ type FileSpec struct { type ConfigureRequest struct { Hostname string `json:"hostname"` RootDevice string `json:"root_device"` + RootFSType string `json:"root_fs_type,omitempty"` + RootSource string `json:"root_source,omitempty"` + VirtioFSTag string `json:"virtio_fs_tag,omitempty"` OverlayDevice string `json:"overlay_device"` RootReadonly bool `json:"root_readonly,omitempty"` Process ProcessSpec `json:"process"` diff --git a/internal/server/fsmanager_test.go b/internal/server/fsmanager_test.go index 7b802aa..adedeb2 100644 --- a/internal/server/fsmanager_test.go +++ b/internal/server/fsmanager_test.go @@ -58,10 +58,6 @@ func (s *fsTestImageService) ImageProcess(string) (*imageconfig.Process, error) return &imageconfig.Process{}, nil } -func (s *fsTestImageService) RootfsMaterialization(string) (*imageapi.RootfsMaterialization, error) { - return &imageapi.RootfsMaterialization{}, nil -} - func (s *fsTestImageService) UmountOCI(req *imageapi.OCIUmountRequest) error { s.mu.Lock() s.ociUmountCalls[req.ImageURL]++ diff --git a/internal/server/server.go b/internal/server/server.go index 66ecaa5..cc16c2a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -46,7 +46,6 @@ import ( _ "github.com/inclusionAI/sandboxd/pkg/networkmanager/bridge" "github.com/inclusionAI/sandboxd/pkg/resourcemanager" svc "github.com/inclusionAI/sandboxd/pkg/runtime" - "github.com/inclusionAI/sandboxd/pkg/runtime/firecracker" "github.com/inclusionAI/sandboxd/pkg/sandbox" "github.com/inclusionAI/sandboxd/pkg/store" "github.com/inclusionAI/sandboxd/pkg/volumemanager" @@ -91,9 +90,6 @@ type sandboxService struct { xpuMgr *xpumanager.Manager store store.DbStore - // firecrackerOCIConverter is present only when the node explicitly enables - // eager OCI-to-EROFS materialization for Firecracker root filesystems. - firecrackerOCIConverter *firecracker.OCIRootfsConverter runtime.UnimplementedSandboxServiceServer @@ -802,25 +798,6 @@ func NewSandboxService(root, configPath string) (result SandboxService, retErr e } } }() - if cfg.RuntimeConfig.Firecracker.OCIRootfsEnabled { - mkfsEROFS := strings.TrimSpace( - cfg.RuntimeConfig.Firecracker.MkfsEROFSPath, - ) - if mkfsEROFS == "" { - mkfsEROFS = config.DefaultFirecrackerMkfsEROFS - } - converter, converterErr := firecracker.NewOCIRootfsConverter( - mkfsEROFS, - ) - if converterErr != nil { - return nil, fmt.Errorf( - "initialize Firecracker OCI rootfs converter: %w", - converterErr, - ) - } - s.firecrackerOCIConverter = converter - } - s.loadRuntimeHandlers() if nodeResMod != nil && cfg.RuntimeConfig.FilestoreDir != "" { if _, ok := s.serviceHandler.Get(config.RuntimeNameRunsc); ok { @@ -1403,51 +1380,6 @@ func (h *sandboxService) Start(ctx context.Context, request *runtime.StartReques }, err } runtimeRootfs := preparedFilesystem.RootfsPath() - if startReq.Runtime == config.RuntimeNameFirecracker && - startReq.Rootfs.GetType() == runtime.RootfsSrcType_IMAGE { - if h.firecrackerOCIConverter == nil { - err := errors.New( - "Firecracker OCI image rootfs conversion is not configured", - ) - return &runtime.StartResponse{Code: -1, Message: err.Error()}, err - } - if h.imageSvc == nil { - err := errors.New("image manager is unavailable for Firecracker OCI rootfs conversion") - return &runtime.StartResponse{Code: -1, Message: err.Error()}, err - } - materialization, materializationErr := h.imageSvc.RootfsMaterialization( - startReq.Rootfs.GetImageUrl(), - ) - if materializationErr != nil { - return &runtime.StartResponse{ - Code: -1, - Message: fmt.Sprintf( - "failed to resolve Firecracker OCI rootfs metadata: %v", - materializationErr, - ), - }, materializationErr - } - if materialization == nil { - err := errors.New("image manager returned empty Firecracker OCI rootfs metadata") - return &runtime.StartResponse{Code: -1, Message: err.Error()}, err - } - runtimeRootfs, err = h.firecrackerOCIConverter.Convert( - ctx, - startReq.Rootfs.GetImageUrl(), - materialization.ContentID, - materialization.ArtifactDir, - runtimeRootfs, - ) - if err != nil { - return &runtime.StartResponse{ - Code: -1, - Message: fmt.Sprintf( - "failed to prepare Firecracker OCI rootfs: %v", - err, - ), - }, err - } - } var specUpdates *svc.SpecUpdates if len(startReq.XpuAllocations) > 0 { if h.xpuMgr == nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 91d013b..7bcb9f3 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -279,7 +279,7 @@ func TestStartRejectsFirecrackerOCIImageBeforeFilesystemPrepare(t *testing.T) { }, }) assert.Equal(t, codes.InvalidArgument, status.Code(err)) - assert.Contains(t, response.Message, "does not support OCI image rootfs") + assert.Contains(t, response.Message, "OCI image rootfs requires virtio-fs") } func TestStartRejectsXPUForUnsupportedRuntimes(t *testing.T) { diff --git a/pkg/imagemanager/api/http.go b/pkg/imagemanager/api/http.go index 3505d5c..9c985b2 100644 --- a/pkg/imagemanager/api/http.go +++ b/pkg/imagemanager/api/http.go @@ -606,76 +606,6 @@ func (w *HttpWorker) ImageProcess(imageURL string) (*imageconfig.Process, error) return w.ociMgr.ImageProcessWithContext(w.ctx, imageURL) } -// RootfsMaterialization resolves content-addressed storage owned by the -// currently mounted OCI or Nydus image. It is intentionally separate from -// MountOCI so ordinary directory-backed runtimes do not hash Nydus metadata. -func (w *HttpWorker) RootfsMaterialization(imageURL string) (*RootfsMaterialization, error) { - if imageURL == "" { - return nil, fmt.Errorf("image_url is required") - } - - var record *MountRecord - if w.mountStore != nil { - var err error - record, err = w.mountStore.Get(imageURL) - if err != nil { - return nil, fmt.Errorf("read image mount record for %s: %w", imageURL, err) - } - } - if record == nil || record.MountType == "oci" { - if w.ociMgr == nil { - return nil, fmt.Errorf("oci manager is not initialized") - } - contentID, artifactDir, err := w.ociMgr.RootfsMaterialization(imageURL) - if err == nil { - return &RootfsMaterialization{ - ContentID: contentID, - ArtifactDir: artifactDir, - }, nil - } - if record == nil { - return nil, err - } - return nil, fmt.Errorf("resolve OCI rootfs materialization for %s: %w", imageURL, err) - } - if record.MountType != "nydus" { - return nil, fmt.Errorf("unsupported image mount type %q for %s", record.MountType, imageURL) - } - if w.mgr == nil { - return nil, fmt.Errorf("Nydus manager is not initialized") - } - - nydusImageURL := record.NydusImageURL - if nydusImageURL == "" { - nydusImageURL = imageURL - } - daemon := w.mgr.GetDaemon(generateNydusID(nydusImageURL)) - if daemon == nil { - return nil, fmt.Errorf("Nydus daemon for %s is unavailable", nydusImageURL) - } - bootstrap := daemon.BootstrapPath() - if bootstrap == "" { - return nil, fmt.Errorf("Nydus daemon for %s has no bootstrap", nydusImageURL) - } - file, err := os.Open(bootstrap) - if err != nil { - return nil, fmt.Errorf("open Nydus bootstrap %s: %w", bootstrap, err) - } - digester := sha256.New() - _, copyErr := io.Copy(digester, file) - closeErr := file.Close() - if copyErr != nil { - return nil, fmt.Errorf("hash Nydus bootstrap %s: %w", bootstrap, copyErr) - } - if closeErr != nil { - return nil, fmt.Errorf("close Nydus bootstrap %s: %w", bootstrap, closeErr) - } - return &RootfsMaterialization{ - ContentID: "sha256:" + hex.EncodeToString(digester.Sum(nil)), - ArtifactDir: daemon.ArtifactDir(), - }, nil -} - // recordMount persists a mount record. Errors are logged but not propagated. func (w *HttpWorker) recordMount(imageURL, mountType, nydusImageURL, mountPoint string) { if w.mountStore == nil { diff --git a/pkg/imagemanager/api/service.go b/pkg/imagemanager/api/service.go index b613ed3..e494633 100644 --- a/pkg/imagemanager/api/service.go +++ b/pkg/imagemanager/api/service.go @@ -32,7 +32,6 @@ type Service interface { MountOCI(req *OCIMountRequest) (*OCIMountResponse, error) ImageProcess(imageURL string) (*imageconfig.Process, error) UmountOCI(req *OCIUmountRequest) error - RootfsMaterialization(imageURL string) (*RootfsMaterialization, error) MountNydus(req *NydusMountRequest) (*MountInfo, error) UmountNydus(req *NydusUmountRequest) error CleanupDaemon(req *CleanupDaemonRequest) error diff --git a/pkg/imagemanager/api/types.go b/pkg/imagemanager/api/types.go index fa869fe..dfd807e 100644 --- a/pkg/imagemanager/api/types.go +++ b/pkg/imagemanager/api/types.go @@ -57,13 +57,6 @@ type OCIMountResponse struct { ImageProcess *imageconfig.Process `json:"image_process,omitempty"` } -// RootfsMaterialization locates content-addressed, image-owned storage for a -// derived root filesystem artifact. -type RootfsMaterialization struct { - ContentID string - ArtifactDir string -} - // OCIMountRequest is used to request mounting an OCI image type OCIMountRequest struct { ImageURL string `json:"image_url"` // Image URL, e.g., "library/alpine:latest" diff --git a/pkg/imagemanager/oci/manager.go b/pkg/imagemanager/oci/manager.go index 0756f00..8e90bf7 100644 --- a/pkg/imagemanager/oci/manager.go +++ b/pkg/imagemanager/oci/manager.go @@ -738,36 +738,6 @@ func (m *Manager) ListMountedDetails() ([]OciMountRecord, error) { return details, nil } -// RootfsMaterialization identifies the immutable chain backing a mounted OCI -// image and a directory owned by that chain's existing GC lifecycle. Derived -// artifacts placed there are removed with the chain instead of requiring a -// separate cache and reference counter. -func (m *Manager) RootfsMaterialization(imageURL string) (contentID, artifactDir string, err error) { - if imageURL == "" { - return "", "", fmt.Errorf("imageURL is required") - } - - unlockImage := m.acquireImageLock(imageURL) - defer unlockImage() - - info := m.getContainer(imageURL) - if info == nil || len(info.ChainIDs) == 0 { - return "", "", fmt.Errorf("OCI image %s is not mounted", imageURL) - } - chainID := info.ChainIDs[len(info.ChainIDs)-1] - - unlockChain := m.acquireChainLock(chainID) - defer unlockChain() - chain, err := m.store.getChain(chainID) - if err != nil { - return "", "", fmt.Errorf("query OCI chain %s: %w", chainID, err) - } - if chain == nil || chain.Path == "" || !pathExists(chain.Path) { - return "", "", fmt.Errorf("OCI chain %s is unavailable", chainID) - } - return chainID, filepath.Dir(chain.Path), nil -} - // UnmountImageWithContext unmounts an OCI overlay mount and updates layer references. func (m *Manager) UnmountImageWithContext(ctx context.Context, imageURL string) (retErr error) { timing, _ := StartOCITimedOperation(ctx, "oci.UnmountImage", imageURL) diff --git a/pkg/imagemanager/oci/manager_test.go b/pkg/imagemanager/oci/manager_test.go index 25895b7..8fb81e7 100644 --- a/pkg/imagemanager/oci/manager_test.go +++ b/pkg/imagemanager/oci/manager_test.go @@ -500,38 +500,6 @@ func TestReconcileState_FixesChainRefsForRecoveredMount(t *testing.T) { } } -func TestRootfsMaterializationUsesFinalChainLifecycle(t *testing.T) { - mgr := newTestManager(t) - defer mgr.store.close() - - chainPath := filepath.Join(mgr.chainsDir, "content-chain", "fs") - if err := os.MkdirAll(chainPath, 0755); err != nil { - t.Fatal(err) - } - if err := mgr.store.putChain(&ChainRecord{ - ChainID: "sha256:final-chain", - Path: chainPath, - RefCount: 1, - }); err != nil { - t.Fatal(err) - } - mgr.setContainer("registry.example/image:tag", &ContainerInfo{ - ImageURL: "registry.example/image:tag", - ChainIDs: []string{"sha256:base-chain", "sha256:final-chain"}, - }) - - contentID, artifactDir, err := mgr.RootfsMaterialization("registry.example/image:tag") - if err != nil { - t.Fatal(err) - } - if contentID != "sha256:final-chain" { - t.Fatalf("content ID = %q, want final chain", contentID) - } - if artifactDir != filepath.Dir(chainPath) { - t.Fatalf("artifact directory = %q, want %q", artifactDir, filepath.Dir(chainPath)) - } -} - func TestReconcileState_DropsPersistedMountWhenMountMissing(t *testing.T) { mgr := newTestManager(t) defer mgr.store.close() diff --git a/pkg/runtime/firecracker/api.go b/pkg/runtime/firecracker/api.go index 3aa651b..64e60b8 100644 --- a/pkg/runtime/firecracker/api.go +++ b/pkg/runtime/firecracker/api.go @@ -124,6 +124,7 @@ func (api *firecrackerAPI) createSnapshot( ctx context.Context, statePath, memoryPath, + fsStatePath, snapshotType string, ) error { body := map[string]any{ @@ -132,6 +133,9 @@ func (api *firecrackerAPI) createSnapshot( "mem_file_path": memoryPath, "deferred_sync": true, } + if fsStatePath != "" { + body["fs_state_path"] = fsStatePath + } // Checkpoint artifacts deliberately remain in the host page cache. The // caller accepts that success does not imply immediate power-loss // durability; avoiding a forced writeback keeps the pause path short. @@ -142,15 +146,26 @@ func (api *firecrackerAPI) loadSnapshot( ctx context.Context, statePath, memoryPath, + liveMemoryPath, tapName, - vsockPath string, + vsockPath, + virtioFSSocketPath, + virtioFSStatePath string, ) error { - return api.put(ctx, "/snapshot/load", map[string]any{ - "snapshot_path": statePath, - "mem_backend": map[string]string{ - "backend_type": "File", - "backend_path": memoryPath, - }, + backend := map[string]string{ + "backend_type": "File", + "backend_path": memoryPath, + } + if virtioFSSocketPath != "" { + backend = map[string]string{ + "backend_type": "SharedFile", + "backend_path": liveMemoryPath, + "source_path": memoryPath, + } + } + body := map[string]any{ + "snapshot_path": statePath, + "mem_backend": backend, "track_dirty_pages": true, "resume_vm": true, "network_overrides": []map[string]string{{ @@ -160,7 +175,15 @@ func (api *firecrackerAPI) loadSnapshot( "vsock_override": map[string]string{ "uds_path": vsockPath, }, - }) + } + if virtioFSSocketPath != "" { + body["fs_override"] = map[string]string{ + "fs_id": "root", + "socket_path": virtioFSSocketPath, + "state_path": virtioFSStatePath, + } + } + return api.put(ctx, "/snapshot/load", body) } func firecrackerDrivePath(id string) string { @@ -178,6 +201,7 @@ func configureFirecrackerVM( tapName, guestMAC, vsockPath string, + virtioFSSocketPath string, drives []firecrackerDrive, ) error { if err := api.put(ctx, "/boot-source", map[string]any{ @@ -195,6 +219,15 @@ func configureFirecrackerVM( }); err != nil { return err } + if virtioFSSocketPath != "" { + if err := api.put(ctx, "/fs/root", map[string]any{ + "fs_id": "root", + "socket_path": virtioFSSocketPath, + "tag": firecrackerVirtioFSTag, + }); err != nil { + return err + } + } for _, drive := range drives { if err := api.put(ctx, firecrackerDrivePath(drive.ID), map[string]any{ "drive_id": drive.ID, diff --git a/pkg/runtime/firecracker/api_test.go b/pkg/runtime/firecracker/api_test.go index 2d07a24..07e3767 100644 --- a/pkg/runtime/firecracker/api_test.go +++ b/pkg/runtime/firecracker/api_test.go @@ -77,6 +77,7 @@ func TestConfigureFirecrackerVM(t *testing.T) { "tap-test", "02:fc:0a:2a:00:02", "/run/firecracker/vsock", + "", []firecrackerDrive{ {ID: "rootfs", Path: "/images/root.erofs", ReadOnly: true}, {ID: "overlay", Path: "/storage/overlay.ext4"}, @@ -171,6 +172,7 @@ func TestFirecrackerSnapshotAPI(t *testing.T) { ctx, "/tmp/vmstate", "/tmp/memory", + "", firecrackerSnapshotTypeFull, ); err != nil { t.Fatal(err) @@ -182,15 +184,39 @@ func TestFirecrackerSnapshotAPI(t *testing.T) { ctx, "/tmp/vmstate", "/tmp/memory", + "", "tap-restored", "/run/firecracker/restored.vsock", + "", + "", + ); err != nil { + t.Fatal(err) + } + if err := api.createSnapshot( + ctx, + "/checkpoint/vmstate", + "/checkpoint/memory", + "/checkpoint/virtiofs.state", + firecrackerSnapshotTypeIncremental, + ); err != nil { + t.Fatal(err) + } + if err := api.loadSnapshot( + ctx, + "/checkpoint/vmstate", + "/checkpoint/memory", + "/storage/memory.live", + "tap-virtiofs", + "/run/firecracker/virtiofs.vsock", + "/run/firecracker/virtiofs.sock", + "/checkpoint/virtiofs.state", ); err != nil { t.Fatal(err) } mu.Lock() defer mu.Unlock() - if len(calls) != 4 { + if len(calls) != 6 { t.Fatalf("snapshot API calls = %+v", calls) } if calls[0].method != http.MethodPatch || calls[0].path != "/vm" || @@ -222,6 +248,25 @@ func TestFirecrackerSnapshotAPI(t *testing.T) { if vsock["uds_path"] != "/run/firecracker/restored.vsock" { t.Fatalf("vsock override = %+v", vsock) } + memory := calls[3].payload["mem_backend"].(map[string]any) + if memory["backend_type"] != "File" || memory["backend_path"] != "/tmp/memory" { + t.Fatalf("memory backend = %+v", memory) + } + if calls[4].payload["fs_state_path"] != "/checkpoint/virtiofs.state" { + t.Fatalf("virtio-fs snapshot create = %+v", calls[4]) + } + sharedMemory := calls[5].payload["mem_backend"].(map[string]any) + if sharedMemory["backend_type"] != "SharedFile" || + sharedMemory["backend_path"] != "/storage/memory.live" || + sharedMemory["source_path"] != "/checkpoint/memory" { + t.Fatalf("shared memory backend = %+v", sharedMemory) + } + fsOverride := calls[5].payload["fs_override"].(map[string]any) + if fsOverride["fs_id"] != "root" || + fsOverride["socket_path"] != "/run/firecracker/virtiofs.sock" || + fsOverride["state_path"] != "/checkpoint/virtiofs.state" { + t.Fatalf("virtio-fs override = %+v", fsOverride) + } } func TestFirecrackerAPIErrorIncludesBody(t *testing.T) { diff --git a/pkg/runtime/firecracker/artifact.go b/pkg/runtime/firecracker/artifact.go index 3991fdf..a6ec707 100644 --- a/pkg/runtime/firecracker/artifact.go +++ b/pkg/runtime/firecracker/artifact.go @@ -48,6 +48,7 @@ const ( type firecrackerCheckpointCompat struct { Arch string `json:"arch,omitempty"` Firecracker string `json:"firecracker,omitempty"` + VirtioFSD string `json:"virtiofsd,omitempty"` Kernel string `json:"kernel,omitempty"` Initrd string `json:"initrd,omitempty"` Vcpus uint32 `json:"vcpus,omitempty"` @@ -62,6 +63,7 @@ type firecrackerCheckpointManifest struct { SnapshotType string `json:"snapshot_type"` MemorySize int64 `json:"memory_size"` BaseMemory string `json:"base_memory,omitempty"` + VirtioFS bool `json:"virtio_fs,omitempty"` Compat *firecrackerCheckpointCompat `json:"compat,omitempty"` CreatedAt time.Time `json:"created_at"` Digests map[string]string `json:"digests"` @@ -179,6 +181,9 @@ func finalizeFirecrackerCheckpointV2( ) (retErr error) { manifest.Version = firecrackerCheckpointVersion2 manifest.CreatedAt = time.Now().UTC() + if manifest.VirtioFS != (files.VirtioFSState != "") { + return errors.New("Firecracker checkpoint virtio-fs manifest does not match its components") + } if manifest.MemorySize <= 0 { // Full snapshots discover the guest memory size from the file // Firecracker just wrote. @@ -188,11 +193,11 @@ func finalizeFirecrackerCheckpointV2( } manifest.MemorySize = info.Size() } - // Only the small state component is digested. Hashing guest memory or the - // writable overlay costs seconds per GiB of CPU and cache reads, which can - // dominate checkpoint latency. Their integrity rests on the local reflink - // copy-on-write and Firecracker's own writes. - manifest.Digests = make(map[string]string, 1) + // Only the small VM and optional virtio-fs state components are digested. + // Hashing guest memory or the writable overlay costs seconds per GiB of CPU + // and cache reads, which can dominate checkpoint latency. Their integrity + // rests on local reflink copy-on-write and Firecracker's own writes. + manifest.Digests = make(map[string]string, 2) for _, component := range firecrackerCheckpointComponents(files) { if component.name == firecrackerCheckpointMemoryName || component.name == firecrackerCheckpointOverlayName { @@ -261,6 +266,12 @@ func openFirecrackerCheckpoint(dir string) (*firecrackerCheckpointArtifact, erro Overlay: filepath.Join(dir, firecrackerCheckpointOverlayName), }, } + if manifest.VirtioFS { + artifact.Files.VirtioFSState = filepath.Join( + dir, + firecrackerCheckpointVirtioFSName, + ) + } for _, component := range firecrackerCheckpointComponents(artifact.Files) { info, err := os.Lstat(component.path) if err != nil { @@ -327,6 +338,7 @@ func readFirecrackerCheckpointManifest(dir string) (*firecrackerCheckpointManife if manifest.Compat != nil { for name, digest := range map[string]string{ "firecracker": manifest.Compat.Firecracker, + "virtiofsd": manifest.Compat.VirtioFSD, "kernel": manifest.Compat.Kernel, "initrd": manifest.Compat.Initrd, } { @@ -454,11 +466,18 @@ func (cache *checkpointDigestCache) remember( func firecrackerCheckpointComponents( files firecrackerCheckpointFiles, ) []struct{ name, path string } { - return []struct{ name, path string }{ + components := []struct{ name, path string }{ {name: firecrackerCheckpointStateName, path: files.State}, {name: firecrackerCheckpointMemoryName, path: files.Memory}, {name: firecrackerCheckpointOverlayName, path: files.Overlay}, } + if files.VirtioFSState != "" { + components = append(components, struct{ name, path string }{ + name: firecrackerCheckpointVirtioFSName, + path: files.VirtioFSState, + }) + } + return components } func digestFirecrackerCheckpointComponent( diff --git a/pkg/runtime/firecracker/artifact_test.go b/pkg/runtime/firecracker/artifact_test.go index 7dac07a..4222b6e 100644 --- a/pkg/runtime/firecracker/artifact_test.go +++ b/pkg/runtime/firecracker/artifact_test.go @@ -391,3 +391,64 @@ func TestFinalizeCheckpointV2SkipsLargeComponentDigests(t *testing.T) { t.Fatalf("verify Full artifact digests: %v", err) } } + +func TestCheckpointV2CarriesVirtioFSState(t *testing.T) { + dir := t.TempDir() + files := sealArtifactFixture(t, dir) + files.VirtioFSState = filepath.Join(dir, firecrackerCheckpointVirtioFSName) + writeArtifactComponent(t, files.VirtioFSState, 4<<10) + manifest := &firecrackerCheckpointManifest{ + SnapshotType: firecrackerSnapshotTypeSoftDirty, + MemorySize: 64 << 10, + VirtioFS: true, + } + if err := finalizeFirecrackerCheckpointV2( + context.Background(), + files, + manifest, + ); err != nil { + t.Fatalf("finalize virtio-fs checkpoint: %v", err) + } + if _, recorded := manifest.Digests[firecrackerCheckpointVirtioFSName]; !recorded { + t.Fatal("virtio-fs state digest was not recorded") + } + artifact, err := openFirecrackerCheckpoint(dir) + if err != nil { + t.Fatalf("open virtio-fs checkpoint: %v", err) + } + if !artifact.Manifest.VirtioFS || artifact.Files.VirtioFSState != files.VirtioFSState { + t.Fatalf("virtio-fs artifact = %+v", artifact) + } + var cache checkpointDigestCache + if err := cache.verifyFirecrackerCheckpointDigests( + context.Background(), + artifact, + ); err != nil { + t.Fatalf("verify virtio-fs checkpoint: %v", err) + } + if err := os.WriteFile(files.VirtioFSState, []byte("changed"), 0600); err != nil { + t.Fatal(err) + } + if err := cache.verifyFirecrackerCheckpointDigests( + context.Background(), + artifact, + ); err == nil { + t.Fatal("accepted changed virtio-fs state") + } +} + +func TestFinalizeCheckpointV2RejectsVirtioFSMismatch(t *testing.T) { + dir := t.TempDir() + files := sealArtifactFixture(t, dir) + if err := finalizeFirecrackerCheckpointV2( + context.Background(), + files, + &firecrackerCheckpointManifest{ + SnapshotType: firecrackerSnapshotTypeFull, + MemorySize: 64 << 10, + VirtioFS: true, + }, + ); err == nil { + t.Fatal("sealed a virtio-fs manifest without device state") + } +} diff --git a/pkg/runtime/firecracker/checkpoint.go b/pkg/runtime/firecracker/checkpoint.go index ee4c168..90cd2b4 100644 --- a/pkg/runtime/firecracker/checkpoint.go +++ b/pkg/runtime/firecracker/checkpoint.go @@ -34,6 +34,7 @@ const ( firecrackerCheckpointStateName = "vmstate" firecrackerCheckpointMemoryName = "memory" firecrackerCheckpointOverlayName = "overlay.ext4" + firecrackerCheckpointVirtioFSName = "virtiofs.state" firecrackerCheckpointFormatName = ".sandboxd-checkpoint-format" firecrackerCheckpointFormat = "1\n" firecrackerCheckpointMaxComponent = int64(16 << 40) @@ -50,9 +51,10 @@ type firecrackerSparseExtent struct { } type firecrackerCheckpointFiles struct { - State string - Memory string - Overlay string + State string + Memory string + Overlay string + VirtioFSState string } func createFirecrackerCheckpointArchive( diff --git a/pkg/runtime/firecracker/checkpoint_handler.go b/pkg/runtime/firecracker/checkpoint_handler.go index b614b12..9756841 100644 --- a/pkg/runtime/firecracker/checkpoint_handler.go +++ b/pkg/runtime/firecracker/checkpoint_handler.go @@ -68,6 +68,11 @@ func (handler *Handler) Checkpoint( !firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { return fmt.Errorf("Firecracker sandbox %s is not running", sandboxID) } + if state.VirtioFS != nil && + !firecrackerVirtioFSProcessMatches(state.VirtioFS, handler.virtiofsdPath) { + return fmt.Errorf("Firecracker sandbox %s virtiofsd is not running", sandboxID) + } + hasVirtioFS := state.VirtioFS != nil api := newFirecrackerAPI(state.APIPath) requestedType, err := resolveRequestedSnapshotType( @@ -89,7 +94,7 @@ func (handler *Handler) Checkpoint( // Hashing the VMM binary and the guest kernel happens before the pause: // the first checkpoint after a daemon start pays it once, later ones // read the cache. - compat, err := handler.buildCheckpointCompat(state.Vcpus) + compat, err := handler.buildCheckpointCompat(state.Vcpus, hasVirtioFS) if err != nil { return err } @@ -98,6 +103,12 @@ func (handler *Handler) Checkpoint( // work the guest should not wait for. A tier-1/2 layout failure degrades // to a Full snapshot; anything else is unrecoverable. files, err := prepareFirecrackerCheckpointV2(config.Directory, base, layoutMemorySize) + if hasVirtioFS { + files.VirtioFSState = filepath.Join( + config.Directory, + firecrackerCheckpointVirtioFSName, + ) + } if err != nil { if base == "" || layoutMemorySize <= 0 { return fmt.Errorf("lay out Firecracker checkpoint for %s: %w", sandboxID, err) @@ -117,6 +128,12 @@ func (handler *Handler) Checkpoint( if files, err = prepareFirecrackerCheckpointV2(config.Directory, "", 0); err != nil { return fmt.Errorf("lay out Firecracker checkpoint for %s: %w", sandboxID, err) } + if hasVirtioFS { + files.VirtioFSState = filepath.Join( + config.Directory, + firecrackerCheckpointVirtioFSName, + ) + } } tPrepared := time.Now() @@ -225,7 +242,7 @@ func (handler *Handler) Checkpoint( func() error { snapshotAttempted = true return api.createSnapshot( - ctx, files.State, files.Memory, snapshotType, + ctx, files.State, files.Memory, files.VirtioFSState, snapshotType, ) }, ) @@ -292,6 +309,7 @@ func (handler *Handler) Checkpoint( manifest := &firecrackerCheckpointManifest{ SnapshotType: snapshotType, MemorySize: memoryInfo.Size(), + VirtioFS: hasVirtioFS, Compat: compat, } if base != "" { @@ -464,7 +482,10 @@ func selectFirecrackerSnapshotTier( // buildCheckpointCompat assembles the compatibility tuple for a guest with // the given vCPU count, digesting the VMM binary, guest kernel, and initrd // once per handler and caching the results. -func (handler *Handler) buildCheckpointCompat(vcpus uint32) (*firecrackerCheckpointCompat, error) { +func (handler *Handler) buildCheckpointCompat( + vcpus uint32, + withVirtioFS bool, +) (*firecrackerCheckpointCompat, error) { handler.compatMu.Lock() defer handler.compatMu.Unlock() if handler.compatDigests == nil { @@ -483,7 +504,17 @@ func (handler *Handler) buildCheckpointCompat(vcpus uint32) (*firecrackerCheckpo } handler.compatDigests = compat } + if withVirtioFS && handler.compatDigests.VirtioFSD == "" { + digest, err := digestFirecrackerStackFile(handler.virtiofsdPath) + if err != nil { + return nil, fmt.Errorf("digest virtiofsd binary: %w", err) + } + handler.compatDigests.VirtioFSD = digest + } compat := *handler.compatDigests + if !withVirtioFS { + compat.VirtioFSD = "" + } compat.Vcpus = vcpus compat.KernelArgs = handler.kernelArgs return &compat, nil @@ -504,7 +535,10 @@ func (handler *Handler) verifyCheckpointCompat( if recorded == nil { return nil } - local, err := handler.buildCheckpointCompat(recorded.Vcpus) + local, err := handler.buildCheckpointCompat( + recorded.Vcpus, + artifact.Manifest.VirtioFS || recorded.VirtioFSD != "", + ) if err != nil { return err } @@ -512,6 +546,7 @@ func (handler *Handler) verifyCheckpointCompat( for _, field := range []struct{ name, recorded, local string }{ {"arch", recorded.Arch, local.Arch}, {"firecracker", recorded.Firecracker, local.Firecracker}, + {"virtiofsd", recorded.VirtioFSD, local.VirtioFSD}, {"kernel", recorded.Kernel, local.Kernel}, {"initrd", recorded.Initrd, local.Initrd}, {"kernel_args", recorded.KernelArgs, local.KernelArgs}, @@ -553,7 +588,12 @@ func firecrackerBaseMemoryUsable(path string, memorySize int64) bool { // discardUnsealedFirecrackerCheckpoint removes the components of a checkpoint // directory that never reached a manifest; sealed artifacts are left alone. func discardUnsealedFirecrackerCheckpoint(files firecrackerCheckpointFiles) { - for _, path := range []string{files.State, files.Memory, files.Overlay} { + for _, path := range []string{ + files.State, + files.Memory, + files.Overlay, + files.VirtioFSState, + } { if path != "" { _ = os.Remove(path) } @@ -589,10 +629,10 @@ func adoptCheckpointMemory( // instantiateFirecrackerCheckpoint materializes the runtime-side pieces of an // opened checkpoint for a restore and reports the guest memory size it // carries. v1 archives are unpacked into the sandbox state directory; v2 -// directories are restored in place — Firecracker mmaps the artifact's memory -// file, so the caller must keep the checkpoint directory intact for the -// lifetime of the restored sandbox — and only the writable layer is cloned -// into sandbox-owned storage, because the restored VM writes to it. +// directories keep their committed components in place. A conventional VM +// maps the artifact memory privately; a virtio-fs VM asks Firecracker to clone +// it into a sandbox-owned shared live file. The writable layer is always +// cloned into sandbox-owned storage. func instantiateFirecrackerCheckpoint( ctx context.Context, artifact *firecrackerCheckpointArtifact, @@ -628,6 +668,7 @@ func instantiateFirecrackerCheckpoint( } files.State = artifact.Files.State files.Memory = artifact.Files.Memory + files.VirtioFSState = artifact.Files.VirtioFSState // The cloned overlay is a live runtime file, not a durable artifact. // FICLONE makes it immediately usable by Firecracker; syncing here can // force unrelated deferred checkpoint writeback onto restore latency. @@ -665,6 +706,12 @@ func (handler *Handler) Restore( startConfig.CheckpointDir, err, ) } + if artifact.Manifest != nil && artifact.Manifest.VirtioFS && + !handler.virtioFSEnabled { + return errors.New( + "Firecracker checkpoint contains virtio-fs but virtiofs_enabled is false", + ) + } if err := handler.verifyCheckpointCompat(artifact); err != nil { return fmt.Errorf( "refuse Firecracker restore from %s: %w", @@ -699,10 +746,17 @@ func (handler *Handler) Restore( if err != nil { return fmt.Errorf("generate Firecracker restore OCI metadata: %w", err) } - plan, err := prepareFirecrackerStorage(spec, startConfig) + plan, err := prepareFirecrackerStorage(spec, startConfig, handler.virtioFSEnabled) if err != nil { return err } + checkpointHasVirtioFS := artifact.Manifest != nil && artifact.Manifest.VirtioFS + requestedVirtioFS := len(plan.virtioFSExports) > 0 + if checkpointHasVirtioFS != requestedVirtioFS { + return fmt.Errorf( + "Firecracker checkpoint virtio-fs layout does not match the restore rootfs and mounts", + ) + } storageDir, err := createFirecrackerStorageDirectory(handler.storageRoot, startConfig.ID) if err != nil { return err @@ -744,7 +798,9 @@ func (handler *Handler) Restore( runtimeCreated = true apiPath := filepath.Join(runtimeDir, firecrackerAPISocket) vsockPath := filepath.Join(runtimeDir, firecrackerVsock) - if len(apiPath) >= 100 || len(vsockPath) >= 100 { + virtioFSSocketPath := filepath.Join(runtimeDir, firecrackerVirtioFSSocket) + if len(apiPath) >= 100 || len(vsockPath) >= 100 || + len(virtioFSSocketPath) >= 100 { return fmt.Errorf("Firecracker Unix socket path is too long under %s", runtimeDir) } if err := removeFirecrackerSocket(apiPath); err != nil { @@ -753,12 +809,38 @@ func (handler *Handler) Restore( if err := removeFirecrackerSocket(vsockPath); err != nil { return err } + var virtioFSState *firecrackerVirtioFSState + var virtioFSProcess *firecrackerVirtioFSProcess + virtioFSOwned := false + if requestedVirtioFS { + virtioFSState = &firecrackerVirtioFSState{ + SocketPath: virtioFSSocketPath, + SharedDir: filepath.Join(storageDir, firecrackerVirtioFSSharedDir), + } + if err := prepareFirecrackerVirtioFSShared( + virtioFSState.SharedDir, + plan.virtioFSExports, + ); err != nil { + return err + } + defer func() { + if virtioFSOwned { + return + } + retErr = errors.Join( + retErr, + cleanupFirecrackerVirtioFS(virtioFSState, handler.virtiofsdPath), + ) + if virtioFSProcess != nil { + _ = virtioFSProcess.wait() + } + }() + } - // v1 archives are unpacked into the sandbox state directory; v2 - // directories are restored in place — Firecracker mmaps the artifact's - // memory file, so the caller must keep the checkpoint directory intact - // for the lifetime of the restored sandbox. Only the writable layer is - // instantiated into sandbox-owned storage (the restored VM writes to it). + // v1 archives are unpacked into the sandbox state directory. v2 committed + // components stay in the caller-owned directory; virtio-fs restore clones + // memory into sandbox-owned shared live storage, while other restores map + // it privately. The writable layer is always sandbox-owned. tPrepared := time.Now() checkpointFiles, memorySize, err := instantiateFirecrackerCheckpoint( ctx, @@ -788,6 +870,25 @@ func (handler *Handler) Restore( return err } defer stderr.Close() + if virtioFSState != nil { + virtioFSState, virtioFSProcess, err = startFirecrackerVirtioFS( + ctx, + handler.virtiofsdPath, + virtioFSState.SharedDir, + virtioFSState.SocketPath, + stdout, + stderr, + ) + if err != nil { + return err + } + if err := attachFirecrackerVirtioFSProcessGroup( + startConfig.CgroupPath, + virtioFSState.PID, + ); err != nil { + return fmt.Errorf("attach restored virtiofsd to cgroup: %w", err) + } + } command := exec.Command( handler.binary, "--api-sock", apiPath, @@ -814,6 +915,7 @@ func (handler *Handler) Restore( APIPath: apiPath, VsockPath: vsockPath, OverlayPath: checkpointFiles.Overlay, + VirtioFS: virtioFSState, MemoryMiB: uint32(memorySize >> 20), Vcpus: restoredVcpus, CreatedAt: time.Now().Format(time.RFC3339Nano), @@ -824,6 +926,10 @@ func (handler *Handler) Restore( handler.instances[startConfig.ID] = instance handler.mu.Unlock() go handler.waitCommand(instance, command) + if virtioFSProcess != nil { + virtioFSOwned = true + go handler.waitVirtioFS(instance, virtioFSProcess) + } restoreSucceeded := false defer func() { @@ -855,8 +961,11 @@ func (handler *Handler) Restore( ctx, checkpointFiles.State, checkpointFiles.Memory, + filepath.Join(storageDir, firecrackerVirtioFSMemory), startConfig.Network.Interface.Name, vsockPath, + virtioFSSocketPathIfConfigured(virtioFSState), + checkpointFiles.VirtioFSState, ); err != nil { return fmt.Errorf("load Firecracker checkpoint for %s: %w", startConfig.ID, err) } diff --git a/pkg/runtime/firecracker/compat_test.go b/pkg/runtime/firecracker/compat_test.go index e8fca9a..e59d3b7 100644 --- a/pkg/runtime/firecracker/compat_test.go +++ b/pkg/runtime/firecracker/compat_test.go @@ -30,27 +30,30 @@ func stackFixture(t *testing.T) *Handler { binary := filepath.Join(dir, "firecracker") kernel := filepath.Join(dir, "vmlinux") initrd := filepath.Join(dir, "initrd.img") + virtiofsd := filepath.Join(dir, "virtiofsd") for path, content := range map[string]string{ - binary: "vmm-binary", - kernel: "guest-kernel", - initrd: "guest-initrd", + binary: "vmm-binary", + kernel: "guest-kernel", + initrd: "guest-initrd", + virtiofsd: "virtiofsd-binary", } { if err := os.WriteFile(path, []byte(content), 0600); err != nil { t.Fatalf("write stack file %s: %v", path, err) } } return &Handler{ - binary: binary, - kernelPath: kernel, - initrdPath: initrd, - kernelArgs: "console=ttyS0", + binary: binary, + kernelPath: kernel, + initrdPath: initrd, + virtiofsdPath: virtiofsd, + kernelArgs: "console=ttyS0", } } func TestBuildCheckpointCompatDigestsAndCaches(t *testing.T) { handler := stackFixture(t) - first, err := handler.buildCheckpointCompat(2) + first, err := handler.buildCheckpointCompat(2, false) if err != nil { t.Fatalf("build compat: %v", err) } @@ -63,7 +66,7 @@ func TestBuildCheckpointCompatDigestsAndCaches(t *testing.T) { if err := os.WriteFile(handler.kernelPath, []byte("mutated"), 0600); err != nil { t.Fatal(err) } - second, err := handler.buildCheckpointCompat(4) + second, err := handler.buildCheckpointCompat(4, true) if err != nil { t.Fatalf("rebuild compat: %v", err) } @@ -73,6 +76,16 @@ func TestBuildCheckpointCompatDigestsAndCaches(t *testing.T) { if second.Vcpus != 4 { t.Fatalf("vcpu count %d not carried per checkpoint", second.Vcpus) } + if len(second.VirtioFSD) != 64 { + t.Fatalf("virtiofsd digest missing: %+v", second) + } + third, err := handler.buildCheckpointCompat(1, false) + if err != nil { + t.Fatalf("rebuild non-virtio-fs compat: %v", err) + } + if third.VirtioFSD != "" { + t.Fatalf("virtiofsd digest leaked into non-virtio-fs tuple: %+v", third) + } } func TestVerifyCheckpointCompat(t *testing.T) { @@ -106,7 +119,7 @@ func TestVerifyCheckpointCompat(t *testing.T) { t.Fatalf("tuple-less artifact rejected: %v", err) } - matching, err := handler.buildCheckpointCompat(1) + matching, err := handler.buildCheckpointCompat(1, false) if err != nil { t.Fatalf("build compat: %v", err) } diff --git a/pkg/runtime/firecracker/handler.go b/pkg/runtime/firecracker/handler.go index 8adeb03..d9958b6 100644 --- a/pkg/runtime/firecracker/handler.go +++ b/pkg/runtime/firecracker/handler.go @@ -46,6 +46,7 @@ const ( firecrackerStateFilename = "state.json" firecrackerAPISocket = "api.sock" firecrackerVsock = firecrackerproto.HostAgentSocketName + firecrackerVirtioFSSocket = "virtiofs.sock" firecrackerAgentTimeout = 15 * time.Second firecrackerShutdownTimeout = 2 * time.Second // Guest flush budget before pausing for a checkpoint. Syncing a heavily @@ -69,13 +70,14 @@ var ( ) type firecrackerPersistedState struct { - ID string `json:"id"` - PID int `json:"pid"` - BundlePath string `json:"bundle_path"` - APIPath string `json:"api_path"` - VsockPath string `json:"vsock_path"` - OverlayPath string `json:"overlay_path"` - CreatedAt string `json:"created_at"` + ID string `json:"id"` + PID int `json:"pid"` + BundlePath string `json:"bundle_path"` + APIPath string `json:"api_path"` + VsockPath string `json:"vsock_path"` + OverlayPath string `json:"overlay_path"` + VirtioFS *firecrackerVirtioFSState `json:"virtio_fs,omitempty"` + CreatedAt string `json:"created_at"` // MemoryMiB is the guest memory size in MiB, recorded so incremental // checkpoints can preallocate or clone a full-size base memory file. MemoryMiB uint32 `json:"memory_mib,omitempty"` @@ -208,21 +210,20 @@ func (instance *firecrackerInstance) shouldPersist() bool { // Handler manages the Firecracker microVM lifecycle. type Handler struct { - binary string - sandboxRoot string - storageRoot string - runtimeRoot string - kernelPath string - initrdPath string - kernelArgs string - kvmDevice string - defaultVCPUs uint32 - defaultMem uint32 - defaultDisk uint64 - ociLoader runtimecore.OciLoader - // ociRootfsEnabled allows the server-side image preparation path to - // materialize an OCI rootfs directory as EROFS before Start is called. - ociRootfsEnabled bool + binary string + sandboxRoot string + storageRoot string + runtimeRoot string + kernelPath string + initrdPath string + kernelArgs string + kvmDevice string + defaultVCPUs uint32 + defaultMem uint32 + defaultDisk uint64 + ociLoader runtimecore.OciLoader + virtiofsdPath string + virtioFSEnabled bool mu sync.RWMutex instances map[string]*firecrackerInstance @@ -285,9 +286,9 @@ func (handler *Handler) ValidateStartRequest( if rootfs := request.GetRootfs(); rootfs != nil && (rootfs.GetType() == runtimeapi.RootfsSrcType_IMAGE || rootfs.GetImageUrl() != "") { - if !handler.ociRootfsEnabled { + if !handler.virtioFSEnabled { return errors.New( - "Firecracker does not support OCI image rootfs unless conversion is enabled", + "Firecracker OCI image rootfs requires virtio-fs", ) } } @@ -328,6 +329,15 @@ func NewHandler( if err := validateFirecrackerCheckpointMode(firecrackerConfig.CheckpointMode); err != nil { return nil, err } + if firecrackerConfig.VirtioFSEnabled { + if err := validateFirecrackerRegularFile( + firecrackerConfig.VirtioFSDPath, + "virtiofsd binary", + true, + ); err != nil { + return nil, err + } + } if filepath.Clean(firecrackerConfig.KVMDevice) != filepath.Clean(config.DefaultKVMDevice) { return nil, fmt.Errorf( @@ -341,18 +351,6 @@ func NewHandler( if _, err := exec.LookPath("mkfs.ext4"); err != nil { return nil, fmt.Errorf("Firecracker requires mkfs.ext4: %w", err) } - if firecrackerConfig.OCIRootfsEnabled { - mkfsEROFS := strings.TrimSpace(firecrackerConfig.MkfsEROFSPath) - if mkfsEROFS == "" { - mkfsEROFS = config.DefaultFirecrackerMkfsEROFS - } - if _, err := exec.LookPath(mkfsEROFS); err != nil { - return nil, fmt.Errorf( - "Firecracker OCI rootfs conversion requires mkfs.erofs: %w", - err, - ) - } - } sandboxRoot := filepath.Join(cfg.RootDir, "containers") storageRoot := filepath.Join(cfg.RuntimeConfig.FilestoreDir, ".firecracker") for path, mode := range map[string]os.FileMode{ @@ -380,7 +378,8 @@ func NewHandler( checkpointWriteback: newCheckpointWritebackScheduler(), defaultDisk: firecrackerConfig.DefaultOverlaySizeBytes, ociLoader: loader, - ociRootfsEnabled: firecrackerConfig.OCIRootfsEnabled, + virtiofsdPath: firecrackerConfig.VirtioFSDPath, + virtioFSEnabled: firecrackerConfig.VirtioFSEnabled, instances: make(map[string]*firecrackerInstance), } handler.recoverInstances() @@ -412,6 +411,9 @@ func applyFirecrackerDefaults(value *config.FirecrackerConfig) { if value.CheckpointMode == "" { value.CheckpointMode = firecrackerCheckpointModeFull } + if value.VirtioFSDPath == "" { + value.VirtioFSDPath = config.DefaultFirecrackerVirtioFSD + } } func validateFirecrackerCheckpointMode(mode string) error { @@ -497,11 +499,11 @@ func (handler *Handler) Start( if err != nil { return fmt.Errorf("generate Firecracker OCI metadata: %w", err) } - plan, err := prepareFirecrackerStorage(spec, startConfig) + plan, err := prepareFirecrackerStorage(spec, startConfig, handler.virtioFSEnabled) if err != nil { return err } - _, err = createFirecrackerStorageDirectory( + storageDir, err := createFirecrackerStorageDirectory( handler.storageRoot, startConfig.ID, ) @@ -566,7 +568,9 @@ func (handler *Handler) Start( runtimeCreated = true apiPath := filepath.Join(runtimeDir, firecrackerAPISocket) vsockPath := filepath.Join(runtimeDir, firecrackerVsock) - if len(apiPath) >= 100 || len(vsockPath) >= 100 { + virtioFSSocketPath := filepath.Join(runtimeDir, firecrackerVirtioFSSocket) + if len(apiPath) >= 100 || len(vsockPath) >= 100 || + len(virtioFSSocketPath) >= 100 { return fmt.Errorf("Firecracker Unix socket path is too long under %s", runtimeDir) } if err := removeFirecrackerSocket(apiPath); err != nil { @@ -575,6 +579,33 @@ func (handler *Handler) Start( if err := removeFirecrackerSocket(vsockPath); err != nil { return err } + var virtioFSState *firecrackerVirtioFSState + var virtioFSProcess *firecrackerVirtioFSProcess + virtioFSOwned := false + if len(plan.virtioFSExports) > 0 { + virtioFSState = &firecrackerVirtioFSState{ + SocketPath: virtioFSSocketPath, + SharedDir: filepath.Join(storageDir, firecrackerVirtioFSSharedDir), + } + if err := prepareFirecrackerVirtioFSShared( + virtioFSState.SharedDir, + plan.virtioFSExports, + ); err != nil { + return err + } + defer func() { + if virtioFSOwned { + return + } + retErr = errors.Join( + retErr, + cleanupFirecrackerVirtioFS(virtioFSState, handler.virtiofsdPath), + ) + if virtioFSProcess != nil { + _ = virtioFSProcess.wait() + } + }() + } stdout, err := openFirecrackerOutput(startConfig.Stdout) if err != nil { @@ -586,6 +617,25 @@ func (handler *Handler) Start( return err } defer stderr.Close() + if virtioFSState != nil { + virtioFSState, virtioFSProcess, err = startFirecrackerVirtioFS( + ctx, + handler.virtiofsdPath, + virtioFSState.SharedDir, + virtioFSState.SocketPath, + stdout, + stderr, + ) + if err != nil { + return err + } + if err := attachFirecrackerVirtioFSProcessGroup( + startConfig.CgroupPath, + virtioFSState.PID, + ); err != nil { + return fmt.Errorf("attach virtiofsd to cgroup: %w", err) + } + } command := exec.Command( handler.binary, @@ -607,6 +657,7 @@ func (handler *Handler) Start( APIPath: apiPath, VsockPath: vsockPath, OverlayPath: overlayPath, + VirtioFS: virtioFSState, CreatedAt: time.Now().Format(time.RFC3339Nano), }, done: make(chan struct{}), @@ -615,6 +666,10 @@ func (handler *Handler) Start( handler.instances[startConfig.ID] = instance handler.mu.Unlock() go handler.waitCommand(instance, command) + if virtioFSProcess != nil { + virtioFSOwned = true + go handler.waitVirtioFS(instance, virtioFSProcess) + } startSucceeded := false defer func() { @@ -647,14 +702,17 @@ func (handler *Handler) Start( } instance.setMemoryMiB(memoryMiB) instance.setVcpus(vcpus) - drives := []firecrackerDrive{ - plan.rootDrive, - { + drives := make([]firecrackerDrive, 0, 2+len(plan.mountDrives)) + if plan.rootDrive.Path != "" { + drives = append(drives, plan.rootDrive) + } + drives = append(drives, + firecrackerDrive{ ID: "overlay", Path: firecrackerCheckpointOverlayName, ReadOnly: false, }, - } + ) drives = append(drives, plan.mountDrives...) if err := configureFirecrackerVM( bootCtx, @@ -667,6 +725,7 @@ func (handler *Handler) Start( startConfig.Network.Interface.Name, startConfig.Network.GuestHardwareAddr().String(), vsockPath, + virtioFSSocketPathIfConfigured(virtioFSState), drives, ); err != nil { return err @@ -906,6 +965,10 @@ func (handler *Handler) waitCommand( command *exec.Cmd, ) { err := command.Wait() + state := instance.snapshot() + if cleanupErr := cleanupFirecrackerVirtioFS(state.VirtioFS, handler.virtiofsdPath); cleanupErr != nil { + logrus.Warnf("firecracker: clean virtio-fs after VMM exit: %v", cleanupErr) + } select { case <-instance.done: return @@ -919,6 +982,28 @@ func (handler *Handler) waitCommand( } } +func (handler *Handler) waitVirtioFS( + instance *firecrackerInstance, + process *firecrackerVirtioFSProcess, +) { + err := process.wait() + select { + case <-instance.done: + return + default: + } + state := instance.snapshot() + if !firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { + return + } + logrus.Errorf( + "firecracker: virtiofsd for sandbox %s exited unexpectedly: %v", + state.ID, + err, + ) + _ = signalFirecrackerProcess(state, handler.binary, syscall.SIGKILL) +} + func (handler *Handler) waitGuest( instance *firecrackerInstance, ) { @@ -965,6 +1050,11 @@ func (handler *Handler) stopInstance( force bool, ) { state := instance.snapshot() + defer func() { + if err := cleanupFirecrackerVirtioFS(state.VirtioFS, handler.virtiofsdPath); err != nil { + logrus.Warnf("firecracker: clean virtio-fs for %s: %v", state.ID, err) + } + }() if !firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { instance.finish(runtimecore.Exit{ExitedAt: time.Now(), ExitCode: state.ExitCode}) return @@ -1110,6 +1200,19 @@ func (handler *Handler) validatePersistedState( filepath.Join(storageDirectory, "overlay.ext4"), }, } + if state.VirtioFS != nil { + expected["virtio-fs socket"] = [2]string{ + filepath.Clean(state.VirtioFS.SocketPath), + filepath.Join(runtimeDirectory, firecrackerVirtioFSSocket), + } + expected["virtio-fs shared directory"] = [2]string{ + filepath.Clean(state.VirtioFS.SharedDir), + filepath.Join(storageDirectory, firecrackerVirtioFSSharedDir), + } + if state.VirtioFS.PID <= 1 { + return errors.New("Firecracker state has an invalid virtiofsd PID") + } + } for description, paths := range expected { if paths[0] != paths[1] { return fmt.Errorf( @@ -1164,10 +1267,21 @@ func (handler *Handler) recoverState( instance.finish(runtimecore.Exit{ExitedAt: exitTime, ExitCode: state.ExitCode}) if firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { go handler.stopInstance(instance, true) + } else if err := cleanupFirecrackerVirtioFS( + state.VirtioFS, + handler.virtiofsdPath, + ); err != nil { + logrus.Warnf("firecracker: clean exited virtio-fs for %s: %v", state.ID, err) } return instance } if !firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { + if err := cleanupFirecrackerVirtioFS( + state.VirtioFS, + handler.virtiofsdPath, + ); err != nil { + logrus.Warnf("firecracker: clean orphaned virtio-fs for %s: %v", state.ID, err) + } exitTime, _ := time.Parse(time.RFC3339Nano, state.ExitedAt) if exitTime.IsZero() { exitTime = time.Now() @@ -1175,6 +1289,25 @@ func (handler *Handler) recoverState( instance.finish(runtimecore.Exit{ExitedAt: exitTime, ExitCode: 255}) return instance } + if state.VirtioFS != nil && + !firecrackerVirtioFSProcessMatches(state.VirtioFS, handler.virtiofsdPath) { + logrus.Warnf( + "firecracker: terminate sandbox %s because virtiofsd is unavailable", + state.ID, + ) + _ = signalFirecrackerProcess(state, handler.binary, syscall.SIGKILL) + if err := cleanupFirecrackerVirtioFS( + state.VirtioFS, + handler.virtiofsdPath, + ); err != nil { + logrus.Warnf("firecracker: clean missing virtio-fs for %s: %v", state.ID, err) + } + instance.finish(runtimecore.Exit{ExitedAt: time.Now(), ExitCode: 255}) + if err := handler.persistInstance(instance); err != nil { + logrus.Warnf("firecracker: persist missing virtiofsd state: %v", err) + } + return instance + } if !state.Configured { logrus.Warnf( "firecracker: terminate incomplete sandbox %s pid=%d", @@ -1182,6 +1315,12 @@ func (handler *Handler) recoverState( state.PID, ) _ = signalFirecrackerProcess(state, handler.binary, syscall.SIGKILL) + if err := cleanupFirecrackerVirtioFS( + state.VirtioFS, + handler.virtiofsdPath, + ); err != nil { + logrus.Warnf("firecracker: clean incomplete virtio-fs for %s: %v", state.ID, err) + } instance.finish(runtimecore.Exit{ExitedAt: time.Now(), ExitCode: 255}) if err := handler.persistInstance(instance); err != nil { logrus.Warnf("firecracker: persist incomplete state: %v", err) @@ -1218,6 +1357,10 @@ func (handler *Handler) monitorRecovered( defer ticker.Stop() for range ticker.C { state := instance.snapshot() + if state.VirtioFS != nil && + !firecrackerVirtioFSProcessMatches(state.VirtioFS, handler.virtiofsdPath) { + _ = signalFirecrackerProcess(state, handler.binary, syscall.SIGKILL) + } if firecrackerProcessMatches(state.PID, handler.binary, state.APIPath, state.ID) { continue } @@ -1227,6 +1370,12 @@ func (handler *Handler) monitorRecovered( logrus.Warnf("firecracker: persist recovered exit state: %v", err) } } + if err := cleanupFirecrackerVirtioFS( + state.VirtioFS, + handler.virtiofsdPath, + ); err != nil { + logrus.Warnf("firecracker: clean recovered virtio-fs for %s: %v", state.ID, err) + } return } } diff --git a/pkg/runtime/firecracker/handler_test.go b/pkg/runtime/firecracker/handler_test.go index fb1caee..734a3cc 100644 --- a/pkg/runtime/firecracker/handler_test.go +++ b/pkg/runtime/firecracker/handler_test.go @@ -113,7 +113,7 @@ func TestFirecrackerValidateStartRequestRejectsOCIImagesByDefault(t *testing.T) ImageUrl: "example.invalid/rootfs:latest", }, }}, - message: "does not support OCI image rootfs", + message: "OCI image rootfs requires virtio-fs", }, { name: "mount", @@ -144,8 +144,8 @@ func TestFirecrackerValidateStartRequestRejectsOCIImagesByDefault(t *testing.T) } } -func TestFirecrackerValidateStartRequestAllowsEnabledOCIRootfs(t *testing.T) { - handler := &Handler{ociRootfsEnabled: true} +func TestFirecrackerValidateStartRequestAllowsVirtioFSOCIRootfs(t *testing.T) { + handler := &Handler{virtioFSEnabled: true} request := &runtimeapi.StartRequest{Rootfs: &runtimeapi.RootfsConfig{ Type: runtimeapi.RootfsSrcType_IMAGE, Source: &runtimeapi.RootfsConfig_ImageUrl{ @@ -181,6 +181,31 @@ func TestFirecrackerValidateStartRequestRejectsNativeWritableMountOverlap(t *tes } } +func TestFirecrackerValidateStartRequestAllowsVirtioFSRootfsButRejectsImageMount(t *testing.T) { + handler := &Handler{virtioFSEnabled: true} + request := &runtimeapi.StartRequest{ + Rootfs: &runtimeapi.RootfsConfig{ + Type: runtimeapi.RootfsSrcType_IMAGE, + Source: &runtimeapi.RootfsConfig_ImageUrl{ + ImageUrl: "example.invalid/rootfs:latest", + }, + }, + } + if err := handler.ValidateStartRequest(request); err != nil { + t.Fatalf("ValidateStartRequest() error = %v", err) + } + request.Mounts = []*runtimeapi.Mount{{ + Target: "/mnt/image", + Source: &runtimeapi.Mount_ImageUrl{ + ImageUrl: "example.invalid/data:latest", + }, + }} + if err := handler.ValidateStartRequest(request); err == nil || + !strings.Contains(err.Error(), "does not support OCI image mount") { + t.Fatalf("ValidateStartRequest() image mount error = %v", err) + } +} + func TestFirecrackerRuntimeDirectoryIsStableAndBounded(t *testing.T) { handler := &Handler{runtimeRoot: "/run/sandboxd/firecracker"} sandboxID := "sbox-" + strings.Repeat("a", 120) @@ -195,6 +220,7 @@ func TestFirecrackerRuntimeDirectoryIsStableAndBounded(t *testing.T) { for _, socket := range []string{ filepath.Join(first, firecrackerAPISocket), filepath.Join(first, firecrackerVsock), + filepath.Join(first, firecrackerVirtioFSSocket), } { if len(socket) >= 100 { t.Fatalf("socket path is too long: %d bytes: %s", len(socket), socket) @@ -231,6 +257,42 @@ func TestValidateFirecrackerPersistedState(t *testing.T) { if err := handler.validatePersistedState(sandboxID, bundlePath, valid); err != nil { t.Fatalf("valid state rejected: %v", err) } + withVirtioFS := valid + withVirtioFS.VirtioFS = &firecrackerVirtioFSState{ + PID: 1234, + SocketPath: filepath.Join(runtimePath, firecrackerVirtioFSSocket), + SharedDir: filepath.Join(storagePath, firecrackerVirtioFSSharedDir), + } + if err := handler.validatePersistedState( + sandboxID, + bundlePath, + withVirtioFS, + ); err != nil { + t.Fatalf("valid virtio-fs state rejected: %v", err) + } + for name, mutate := range map[string]func(*firecrackerVirtioFSState){ + "PID": func(state *firecrackerVirtioFSState) { state.PID = 1 }, + "socket": func(state *firecrackerVirtioFSState) { + state.SocketPath = filepath.Join(root, "other.sock") + }, + "shared directory": func(state *firecrackerVirtioFSState) { + state.SharedDir = filepath.Join(root, "other") + }, + } { + t.Run("virtio-fs "+name, func(t *testing.T) { + state := withVirtioFS + virtioFS := *withVirtioFS.VirtioFS + state.VirtioFS = &virtioFS + mutate(state.VirtioFS) + if err := handler.validatePersistedState( + sandboxID, + bundlePath, + state, + ); err == nil { + t.Fatalf("accepted inconsistent virtio-fs %s state: %+v", name, state) + } + }) + } tests := []struct { name string diff --git a/pkg/runtime/firecracker/oci_rootfs.go b/pkg/runtime/firecracker/oci_rootfs.go deleted file mode 100644 index 788f185..0000000 --- a/pkg/runtime/firecracker/oci_rootfs.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) 2026 Ant Group Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package firecracker - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/sirupsen/logrus" - "golang.org/x/sync/singleflight" -) - -// OCIRootfsConverter materializes an image-manager-mounted OCI or Nydus rootfs -// directory as an immutable EROFS image suitable for a Firecracker drive. -type OCIRootfsConverter struct { - mkfsPath string - group singleflight.Group -} - -// NewOCIRootfsConverter validates and prepares an OCI-to-EROFS converter. -func NewOCIRootfsConverter(mkfsPath string) (*OCIRootfsConverter, error) { - mkfsPath = strings.TrimSpace(mkfsPath) - if mkfsPath == "" { - return nil, errors.New("Firecracker mkfs.erofs path is empty") - } - resolvedMkfs, err := exec.LookPath(mkfsPath) - if err != nil { - return nil, fmt.Errorf("find mkfs.erofs %q: %w", mkfsPath, err) - } - return &OCIRootfsConverter{ - mkfsPath: resolvedMkfs, - }, nil -} - -// Convert returns a content-addressed EROFS image for imageRef, building it -// atomically under storage owned by the source image's existing lifecycle. -func (converter *OCIRootfsConverter) Convert( - ctx context.Context, - imageRef, - contentID, - artifactDir, - sourceDir string, -) (string, error) { - imageRef = strings.TrimSpace(imageRef) - if imageRef == "" { - return "", errors.New("Firecracker OCI image reference is empty") - } - contentID = strings.TrimSpace(contentID) - if contentID == "" { - return "", errors.New("Firecracker OCI rootfs content ID is empty") - } - artifactDir = strings.TrimSpace(artifactDir) - if artifactDir == "" { - return "", errors.New("Firecracker OCI rootfs artifact directory is empty") - } - artifactInfo, err := os.Stat(artifactDir) - if err != nil { - return "", fmt.Errorf("stat Firecracker OCI artifact directory %s: %w", artifactDir, err) - } - if !artifactInfo.IsDir() { - return "", fmt.Errorf("Firecracker OCI artifact path %s is not a directory", artifactDir) - } - info, err := os.Stat(sourceDir) - if err != nil { - return "", fmt.Errorf("stat Firecracker OCI rootfs %s: %w", sourceDir, err) - } - if !info.IsDir() { - return "", fmt.Errorf("Firecracker OCI rootfs %s is not a directory", sourceDir) - } - - digest := sha256.Sum256([]byte(contentID)) - key := hex.EncodeToString(digest[:]) - groupKey := filepath.Clean(artifactDir) + "\x00" + key - value, err, _ := converter.group.Do(groupKey, func() (interface{}, error) { - return converter.convert(ctx, imageRef, key, artifactDir, sourceDir) - }) - if err != nil { - return "", err - } - return value.(string), nil -} - -func (converter *OCIRootfsConverter) convert( - ctx context.Context, - imageRef, - key, - artifactDir, - sourceDir string, -) (string, error) { - destination := filepath.Join(artifactDir, "rootfs-"+key+".erofs") - if validEROFSFile(destination) { - logrus.Infof( - "reusing Firecracker OCI EROFS image %s for %s", - destination, - imageRef, - ) - return destination, nil - } - if err := os.Remove(destination); err != nil && !os.IsNotExist(err) { - return "", fmt.Errorf("remove invalid Firecracker OCI EROFS cache %s: %w", destination, err) - } - - temporary, err := os.CreateTemp(artifactDir, ".rootfs-"+key+"-*.erofs") - if err != nil { - return "", fmt.Errorf("create Firecracker OCI EROFS temporary file: %w", err) - } - temporaryPath := temporary.Name() - if err := temporary.Close(); err != nil { - _ = os.Remove(temporaryPath) - return "", fmt.Errorf("close Firecracker OCI EROFS temporary file: %w", err) - } - if err := os.Remove(temporaryPath); err != nil { - return "", fmt.Errorf("prepare Firecracker OCI EROFS output path: %w", err) - } - defer os.Remove(temporaryPath) - - started := time.Now() - var stderr bytes.Buffer - command := exec.CommandContext( - ctx, - converter.mkfsPath, - "--quiet", - "-Enoinline_data", - temporaryPath, - sourceDir, - ) - command.Stdout = io.Discard - command.Stderr = &stderr - if err := command.Run(); err != nil { - return "", fmt.Errorf( - "materialize Firecracker OCI rootfs %s: %w: %s", - imageRef, - err, - strings.TrimSpace(stderr.String()), - ) - } - if !validEROFSFile(temporaryPath) { - return "", fmt.Errorf( - "materialized Firecracker OCI rootfs %s is not a valid EROFS image", - imageRef, - ) - } - if err := os.Chmod(temporaryPath, 0644); err != nil { - return "", fmt.Errorf("chmod Firecracker OCI EROFS image: %w", err) - } - if err := os.Rename(temporaryPath, destination); err != nil { - return "", fmt.Errorf("publish Firecracker OCI EROFS image: %w", err) - } - logrus.Infof( - "materialized Firecracker OCI rootfs %s at %s in %s", - imageRef, - destination, - time.Since(started), - ) - return destination, nil -} - -func validEROFSFile(path string) bool { - info, err := os.Lstat(path) - if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { - return false - } - file, err := os.Open(path) - if err != nil { - return false - } - defer file.Close() - var magic [4]byte - if _, err := file.ReadAt(magic[:], 1024); err != nil { - return false - } - return binary.LittleEndian.Uint32(magic[:]) == firecrackerEROFSMagic -} diff --git a/pkg/runtime/firecracker/oci_rootfs_test.go b/pkg/runtime/firecracker/oci_rootfs_test.go deleted file mode 100644 index 8b7661f..0000000 --- a/pkg/runtime/firecracker/oci_rootfs_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2026 Ant Group Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package firecracker - -import ( - "context" - "os" - "path/filepath" - "testing" -) - -func TestOCIRootfsConverterCachesMaterializedImage(t *testing.T) { - root := t.TempDir() - source := filepath.Join(root, "rootfs") - artifacts := filepath.Join(root, "artifacts") - if err := os.Mkdir(source, 0755); err != nil { - t.Fatal(err) - } - if err := os.Mkdir(artifacts, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(source, "marker"), []byte("ok"), 0644); err != nil { - t.Fatal(err) - } - counter := filepath.Join(root, "counter") - t.Setenv("FAKE_MKFS_COUNTER", counter) - mkfs := filepath.Join(root, "mkfs.erofs") - script := `#!/bin/sh -set -eu -printf x >> "${FAKE_MKFS_COUNTER}" -dd if=/dev/zero of="$3" bs=2048 count=1 status=none -printf '\342\341\365\340' | dd of="$3" bs=1 seek=1024 conv=notrunc status=none -` - if err := os.WriteFile(mkfs, []byte(script), 0755); err != nil { - t.Fatal(err) - } - converter, err := NewOCIRootfsConverter(mkfs) - if err != nil { - t.Fatal(err) - } - - first, err := converter.Convert( - context.Background(), - "registry.example/image:v1", - "sha256:content", - artifacts, - source, - ) - if err != nil { - t.Fatal(err) - } - second, err := converter.Convert( - context.Background(), - "registry.example/image:updated-tag", - "sha256:content", - artifacts, - source, - ) - if err != nil { - t.Fatal(err) - } - if first != second { - t.Fatalf("cached paths differ: %q and %q", first, second) - } - if !validEROFSFile(first) { - t.Fatalf("cached image %s is not EROFS", first) - } - data, err := os.ReadFile(counter) - if err != nil { - t.Fatal(err) - } - if string(data) != "x" { - t.Fatalf("mkfs invocation count marker = %q, want one invocation", data) - } -} - -func TestOCIRootfsConverterRejectsNonDirectorySource(t *testing.T) { - root := t.TempDir() - mkfs := filepath.Join(root, "mkfs.erofs") - if err := os.WriteFile(mkfs, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil { - t.Fatal(err) - } - converter, err := NewOCIRootfsConverter(mkfs) - if err != nil { - t.Fatal(err) - } - source := filepath.Join(root, "rootfs") - if err := os.WriteFile(source, []byte("not a directory"), 0644); err != nil { - t.Fatal(err) - } - if _, err := converter.Convert( - context.Background(), - "registry.example/image:v1", - "sha256:content", - root, - source, - ); err == nil { - t.Fatal("Convert() succeeded for a non-directory source") - } -} diff --git a/pkg/runtime/firecracker/storage.go b/pkg/runtime/firecracker/storage.go index c8d50d7..9c1c83e 100644 --- a/pkg/runtime/firecracker/storage.go +++ b/pkg/runtime/firecracker/storage.go @@ -32,6 +32,7 @@ import ( const ( firecrackerEROFSMagic = uint32(0xe0f5e1e2) + firecrackerVirtioFSTag = "sandboxfs" firecrackerMaxInjectedFile = 1 << 20 firecrackerMaxInjectedTotal = 4 << 20 firecrackerMinimumOverlay = 16 << 20 @@ -45,22 +46,25 @@ type firecrackerDrive struct { } type firecrackerStoragePlan struct { - rootDrive firecrackerDrive - mountDrives []firecrackerDrive - configure firecrackerproto.ConfigureRequest + rootDrive firecrackerDrive + mountDrives []firecrackerDrive + virtioFSExports []firecrackerVirtioFSExport + configure firecrackerproto.ConfigureRequest +} + +type firecrackerVirtioFSExport struct { + Source string + RelativePath string } func prepareFirecrackerStorage( spec *runtimecore.Spec, startConfig runtimecore.StartConfig, + virtioFSEnabled bool, ) (*firecrackerStoragePlan, error) { if spec == nil || spec.Root == nil || spec.Root.Path == "" { return nil, errors.New("Firecracker rootfs is missing") } - rootPath, err := validateEROFSImage(spec.Root.Path) - if err != nil { - return nil, fmt.Errorf("validate Firecracker rootfs: %w", err) - } if spec.Process == nil || len(spec.Process.Args) == 0 { return nil, errors.New("Firecracker sandbox process is missing") } @@ -105,16 +109,9 @@ func prepareFirecrackerStorage( return nil, err } plan := &firecrackerStoragePlan{ - rootDrive: firecrackerDrive{ - ID: "rootfs", - Path: rootPath, - ReadOnly: true, - }, configure: firecrackerproto.ConfigureRequest{ - Hostname: spec.Hostname, - RootDevice: "/dev/vda", - OverlayDevice: "/dev/vdb", - RootReadonly: spec.Root.Readonly, + Hostname: spec.Hostname, + RootReadonly: spec.Root.Readonly, Process: firecrackerproto.ProcessSpec{ Args: append([]string(nil), spec.Process.Args...), Env: append([]string(nil), spec.Process.Env...), @@ -138,6 +135,41 @@ func prepareFirecrackerStorage( firecrackerproto.NativeWritableMountSpec{Target: mount.Target}, ) } + nextDrive := 0 + rootInfo, err := os.Stat(spec.Root.Path) + if err != nil { + return nil, fmt.Errorf("validate Firecracker rootfs: %w", err) + } + if rootInfo.IsDir() { + if !virtioFSEnabled { + return nil, fmt.Errorf( + "validate Firecracker rootfs: %s is not a regular EROFS image", + spec.Root.Path, + ) + } + rootPath, err := validateFirecrackerDirectory(spec.Root.Path) + if err != nil { + return nil, fmt.Errorf("validate Firecracker rootfs: %w", err) + } + plan.virtioFSExports = append(plan.virtioFSExports, firecrackerVirtioFSExport{ + Source: rootPath, RelativePath: "rootfs", + }) + plan.configure.RootFSType = "virtiofs" + plan.configure.RootSource = "rootfs" + } else { + rootPath, err := validateEROFSImage(spec.Root.Path) + if err != nil { + return nil, fmt.Errorf("validate Firecracker rootfs: %w", err) + } + plan.rootDrive = firecrackerDrive{ + ID: "rootfs", Path: rootPath, ReadOnly: true, + } + plan.configure.RootFSType = "erofs" + plan.configure.RootDevice = firecrackerGuestBlockDevice(nextDrive) + nextDrive++ + } + plan.configure.OverlayDevice = firecrackerGuestBlockDevice(nextDrive) + nextDrive++ injectedBytes := 0 for _, mount := range mounts { @@ -165,6 +197,42 @@ func prepareFirecrackerStorage( }, ) case "bind": + if err := validateFirecrackerReadOnlyBind(mount); err != nil { + return nil, err + } + info, err := os.Stat(mount.Source) + if err != nil { + return nil, err + } + if info.IsDir() { + if !virtioFSEnabled { + return nil, fmt.Errorf( + "Firecracker only supports regular-file bind injection, got %s", + mount.Source, + ) + } + source, err := validateFirecrackerDirectory(mount.Source) + if err != nil { + return nil, err + } + options, err := firecrackerVirtioFSMountOptions(mount.Options) + if err != nil { + return nil, fmt.Errorf("validate Firecracker mount %s: %w", target, err) + } + relative := fmt.Sprintf("mounts/%04d", len(plan.virtioFSExports)) + plan.virtioFSExports = append( + plan.virtioFSExports, + firecrackerVirtioFSExport{Source: source, RelativePath: relative}, + ) + plan.configure.Mounts = append( + plan.configure.Mounts, + firecrackerproto.MountSpec{ + Source: relative, Target: target, + FSType: "virtiofs", Options: options, + }, + ) + break + } file, size, err := firecrackerInjectedFile(mount) if err != nil { return nil, err @@ -187,7 +255,7 @@ func prepareFirecrackerStorage( ) } index := len(plan.mountDrives) - if index+2 >= firecrackerMaximumDriveCount { + if nextDrive >= firecrackerMaximumDriveCount { return nil, fmt.Errorf( "Firecracker supports at most %d attached drives", firecrackerMaximumDriveCount, @@ -201,12 +269,13 @@ func prepareFirecrackerStorage( plan.configure.Mounts = append( plan.configure.Mounts, firecrackerproto.MountSpec{ - Device: firecrackerGuestBlockDevice(index + 2), + Device: firecrackerGuestBlockDevice(nextDrive), Target: target, FSType: "erofs", Options: firecrackerMountOptions(mount.Options), }, ) + nextDrive++ default: return nil, fmt.Errorf( "Firecracker does not support mount type %q at %s", @@ -215,6 +284,9 @@ func prepareFirecrackerStorage( ) } } + if len(plan.virtioFSExports) > 0 { + plan.configure.VirtioFSTag = firecrackerVirtioFSTag + } return plan, nil } @@ -278,18 +350,8 @@ func validateFirecrackerTmpfsOptions(options []string) ([]string, error) { } func firecrackerInjectedFile(mount runtimecore.Mount) (firecrackerproto.FileSpec, int, error) { - if mount.Source == "" { - return firecrackerproto.FileSpec{}, 0, fmt.Errorf( - "Firecracker bind mount %s has no source", - mount.Destination, - ) - } - if !slices.Contains(mount.Options, "ro") || - slices.Contains(mount.Options, "rw") { - return firecrackerproto.FileSpec{}, 0, fmt.Errorf( - "Firecracker bind mount %s must be explicitly read-only", - mount.Destination, - ) + if err := validateFirecrackerReadOnlyBind(mount); err != nil { + return firecrackerproto.FileSpec{}, 0, err } info, err := os.Stat(mount.Source) if err != nil { @@ -320,6 +382,60 @@ func firecrackerInjectedFile(mount runtimecore.Mount) (firecrackerproto.FileSpec }, len(content), nil } +func validateFirecrackerReadOnlyBind(mount runtimecore.Mount) error { + if mount.Source == "" { + return fmt.Errorf( + "Firecracker bind mount %s has no source", + mount.Destination, + ) + } + if !slices.Contains(mount.Options, "ro") || + slices.Contains(mount.Options, "rw") { + return fmt.Errorf( + "Firecracker bind mount %s must be explicitly read-only", + mount.Destination, + ) + } + return nil +} + +func validateFirecrackerDirectory(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + resolved, err = filepath.Abs(resolved) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.IsDir() { + return "", fmt.Errorf("%s is not a directory", resolved) + } + return resolved, nil +} + +func firecrackerVirtioFSMountOptions(options []string) ([]string, error) { + result := []string{"ro"} + for _, option := range options { + switch option { + case "ro", "bind", "rbind", "private", "rprivate": + case "nodev", "noexec", "nosuid": + if !slices.Contains(result, option) { + result = append(result, option) + } + case "rw": + return nil, errors.New("virtio-fs directory mount cannot be writable") + default: + return nil, fmt.Errorf("unsupported virtio-fs mount option %q", option) + } + } + return result, nil +} + func firecrackerMountOptions(options []string) []string { result := make([]string, 0, len(options)+1) for _, option := range options { @@ -342,7 +458,8 @@ func firecrackerMountOptions(options []string) []string { func firecrackerGuestBlockDevice(index int) string { // Firecracker's virtio-mmio block devices are enumerated in API insertion - // order. The root image is vda and the writable layer is vdb. + // order. With an EROFS root the root image is vda and the writable layer is + // vdb; with a virtio-fs root the writable layer is the first drive, vda. return fmt.Sprintf("/dev/vd%c", 'a'+index) } diff --git a/pkg/runtime/firecracker/storage_test.go b/pkg/runtime/firecracker/storage_test.go index 2e6e215..ac4a0af 100644 --- a/pkg/runtime/firecracker/storage_test.go +++ b/pkg/runtime/firecracker/storage_test.go @@ -68,7 +68,7 @@ func TestPrepareFirecrackerStorage(t *testing.T) { Network: firecrackerTestNetwork(), ExtraConfig: `{"nativeWritableMounts":[` + `{"target":"/var/lib/docker"}]}`, - }) + }, false) if err != nil { t.Fatal(err) } @@ -130,6 +130,7 @@ func TestPrepareFirecrackerStorageRejectsNativeWritableMountOverlap(t *testing.T ExtraConfig: `{"nativeWritableMounts":[` + `{"target":"/var/lib/docker"}]}`, }, + false, ) if err == nil || !strings.Contains(err.Error(), "overlaps mount target") { t.Fatalf("overlap error = %v", err) @@ -167,6 +168,7 @@ func TestPrepareFirecrackerStorageUsesLastMountForTarget(t *testing.T) { plan, err := prepareFirecrackerStorage( spec, runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + false, ) if err != nil { t.Fatal(err) @@ -199,6 +201,7 @@ func TestPrepareFirecrackerStorageRejectsWritableBind(t *testing.T) { }}, }, runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + false, ) if err == nil || !strings.Contains(err.Error(), "explicitly read-only") { t.Fatalf("writable bind error = %v", err) @@ -212,12 +215,56 @@ func TestPrepareFirecrackerStorageRejectsDirectoryRoot(t *testing.T) { Process: &runtimecore.Process{Args: []string{"/bin/true"}}, }, runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + false, ) if err == nil || !strings.Contains(err.Error(), "not a regular EROFS image") { t.Fatalf("directory root error = %v", err) } } +func TestPrepareFirecrackerStorageWithVirtioFSDirectories(t *testing.T) { + root := t.TempDir() + mounted := t.TempDir() + plan, err := prepareFirecrackerStorage( + &runtimecore.Spec{ + Root: &runtimecore.Root{Path: root}, + Process: &runtimecore.Process{Args: []string{"/bin/true"}}, + Mounts: []runtimecore.Mount{{ + Type: "bind", + Source: mounted, + Destination: "/opt/runtime", + Options: []string{"rbind", "ro", "noexec", "nosuid"}, + }}, + }, + runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + true, + ) + if err != nil { + t.Fatal(err) + } + if plan.rootDrive.Path != "" || plan.configure.RootDevice != "" || + plan.configure.RootFSType != "virtiofs" || + plan.configure.RootSource != "rootfs" || + plan.configure.OverlayDevice != "/dev/vda" || + plan.configure.VirtioFSTag != firecrackerVirtioFSTag { + t.Fatalf("virtio-fs root plan = %+v", plan) + } + if len(plan.virtioFSExports) != 2 || + plan.virtioFSExports[0].Source != root || + plan.virtioFSExports[0].RelativePath != "rootfs" || + plan.virtioFSExports[1].Source != mounted || + plan.virtioFSExports[1].RelativePath != "mounts/0001" { + t.Fatalf("virtio-fs exports = %+v", plan.virtioFSExports) + } + if len(plan.configure.Mounts) != 1 || + plan.configure.Mounts[0].FSType != "virtiofs" || + plan.configure.Mounts[0].Source != "mounts/0001" || + plan.configure.Mounts[0].Target != "/opt/runtime" || + strings.Join(plan.configure.Mounts[0].Options, ",") != "ro,noexec,nosuid" { + t.Fatalf("virtio-fs guest mounts = %+v", plan.configure.Mounts) + } +} + func TestPrepareFirecrackerStorageRejectsDirectoryBind(t *testing.T) { _, err := prepareFirecrackerStorage( &runtimecore.Spec{ @@ -231,6 +278,7 @@ func TestPrepareFirecrackerStorageRejectsDirectoryBind(t *testing.T) { }}, }, runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + false, ) if err == nil || !strings.Contains(err.Error(), "regular-file") { t.Fatalf("directory bind error = %v", err) @@ -250,6 +298,7 @@ func TestPrepareFirecrackerStorageRejectsUnsafeTmpfsOption(t *testing.T) { }}, }, runtimecore.StartConfig{Network: firecrackerTestNetwork()}, + false, ) if err == nil || !strings.Contains(err.Error(), "unsupported tmpfs option") { t.Fatalf("unsafe tmpfs error = %v", err) diff --git a/pkg/runtime/firecracker/virtiofs.go b/pkg/runtime/firecracker/virtiofs.go new file mode 100644 index 0000000..0bb472f --- /dev/null +++ b/pkg/runtime/firecracker/virtiofs.go @@ -0,0 +1,442 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firecracker + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const ( + firecrackerVirtioFSSharedDir = "virtiofs" + firecrackerVirtioFSMemory = "memory.live" + firecrackerVirtioFSStartup = 15 * time.Second +) + +type firecrackerVirtioFSState struct { + PID int `json:"pid"` + SocketPath string `json:"socket_path"` + SharedDir string `json:"shared_dir"` +} + +type firecrackerVirtioFSProcess struct { + command *exec.Cmd + done chan struct{} + err error +} + +func waitFirecrackerVirtioFSCommand(command *exec.Cmd) *firecrackerVirtioFSProcess { + process := &firecrackerVirtioFSProcess{ + command: command, + done: make(chan struct{}), + } + go func() { + process.err = command.Wait() + close(process.done) + }() + return process +} + +func (process *firecrackerVirtioFSProcess) wait() error { + <-process.done + return process.err +} + +func virtioFSSocketPathIfConfigured(state *firecrackerVirtioFSState) string { + if state == nil { + return "" + } + return state.SocketPath +} + +func prepareFirecrackerVirtioFSShared( + sharedDir string, + exports []firecrackerVirtioFSExport, +) (retErr error) { + if len(exports) == 0 { + return errors.New("virtio-fs export list is empty") + } + if err := os.Mkdir(sharedDir, 0700); err != nil { + return fmt.Errorf("create virtio-fs shared directory: %w", err) + } + mounted := false + defer func() { + if retErr == nil { + return + } + if mounted { + unmountErr := unmountFirecrackerVirtioFSShared(sharedDir) + retErr = errors.Join(retErr, unmountErr) + if unmountErr != nil { + return + } + } + retErr = errors.Join(retErr, os.RemoveAll(sharedDir)) + }() + if err := unix.Mount( + "tmpfs", + sharedDir, + "tmpfs", + unix.MS_NOSUID|unix.MS_NODEV, + "mode=0700,size=1m", + ); err != nil { + return fmt.Errorf("mount virtio-fs staging tmpfs: %w", err) + } + mounted = true + if err := unix.Mount("", sharedDir, "", unix.MS_PRIVATE|unix.MS_REC, ""); err != nil { + return fmt.Errorf("make virtio-fs staging mount private: %w", err) + } + + seen := make(map[string]struct{}, len(exports)) + for _, export := range exports { + target, err := firecrackerVirtioFSExportPath(sharedDir, export.RelativePath) + if err != nil { + return err + } + if _, exists := seen[target]; exists { + return fmt.Errorf("duplicate virtio-fs export path %q", export.RelativePath) + } + seen[target] = struct{}{} + source, err := validateFirecrackerDirectory(export.Source) + if err != nil { + return fmt.Errorf("validate virtio-fs export %s: %w", export.Source, err) + } + if err := os.MkdirAll(target, 0700); err != nil { + return fmt.Errorf("create virtio-fs export target %s: %w", target, err) + } + if err := unix.Mount(source, target, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { + return fmt.Errorf("bind virtio-fs export %s: %w", source, err) + } + if err := unix.Mount( + "", + target, + "", + unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NODEV, + "", + ); err != nil { + return fmt.Errorf("remount virtio-fs export %s read-only: %w", source, err) + } + } + return nil +} + +func firecrackerVirtioFSExportPath(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) { + return "", fmt.Errorf("invalid virtio-fs export path %q", relative) + } + clean := filepath.Clean(relative) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") { + return "", fmt.Errorf("virtio-fs export path %q escapes its root", relative) + } + target := filepath.Join(root, clean) + rel, err := filepath.Rel(root, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, "../") { + return "", fmt.Errorf("virtio-fs export path %q escapes its root", relative) + } + return target, nil +} + +func startFirecrackerVirtioFS( + ctx context.Context, + binary, + sharedDir, + socketPath string, + stdout, + stderr io.Writer, +) (*firecrackerVirtioFSState, *firecrackerVirtioFSProcess, error) { + if err := removeFirecrackerSocket(socketPath); err != nil { + return nil, nil, err + } + command := exec.Command( + binary, + "--shared-dir", sharedDir, + "--socket-path", socketPath, + "--readonly", + "--no-announce-submounts", + "--sandbox", "namespace", + "--inode-file-handles=never", + "--migration-mode", "find-paths", + "--migration-on-error", "abort", + ) + command.Stdout = stdout + command.Stderr = stderr + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + return nil, nil, fmt.Errorf("start virtiofsd: %w", err) + } + process := waitFirecrackerVirtioFSCommand(command) + waitCtx, cancel := context.WithTimeout(ctx, firecrackerVirtioFSStartup) + defer cancel() + state := &firecrackerVirtioFSState{ + PID: command.Process.Pid, + SocketPath: socketPath, + SharedDir: sharedDir, + } + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + for { + info, err := os.Lstat(socketPath) + if err == nil && info.Mode()&os.ModeSocket != 0 { + return state, process, nil + } + select { + case <-process.done: + if process.err != nil { + return nil, nil, fmt.Errorf( + "virtiofsd exited before creating its socket: %w", + process.err, + ) + } + return nil, nil, errors.New("virtiofsd exited before creating its socket") + case <-waitCtx.Done(): + if group, err := syscall.Getpgid(state.PID); err == nil && group == state.PID { + _ = syscall.Kill(-state.PID, syscall.SIGKILL) + } else { + _ = command.Process.Kill() + } + _ = process.wait() + return nil, nil, fmt.Errorf("wait for virtiofsd socket: %w", waitCtx.Err()) + case <-ticker.C: + } + } +} + +func stopFirecrackerVirtioFS(state *firecrackerVirtioFSState, binary string) { + if state == nil || !firecrackerVirtioFSProcessMatches(state, binary) { + return + } + _ = signalFirecrackerVirtioFS(state, binary, syscall.SIGTERM) + if waitFirecrackerVirtioFS(state, binary, 500*time.Millisecond) { + return + } + _ = signalFirecrackerVirtioFS(state, binary, syscall.SIGKILL) + _ = waitFirecrackerVirtioFS(state, binary, time.Second) +} + +func waitFirecrackerVirtioFS( + state *firecrackerVirtioFSState, + binary string, + timeout time.Duration, +) bool { + deadline := time.Now().Add(timeout) + for { + if !firecrackerVirtioFSProcessMatches(state, binary) { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(20 * time.Millisecond) + } +} + +func signalFirecrackerVirtioFS( + state *firecrackerVirtioFSState, + binary string, + signal syscall.Signal, +) error { + if !firecrackerVirtioFSProcessMatches(state, binary) { + return nil + } + group, err := syscall.Getpgid(state.PID) + if err == nil && group == state.PID { + return syscall.Kill(-state.PID, signal) + } + return syscall.Kill(state.PID, signal) +} + +func firecrackerVirtioFSProcessMatches( + state *firecrackerVirtioFSState, + binary string, +) bool { + if state == nil || state.PID <= 1 || syscall.Kill(state.PID, 0) != nil { + return false + } + stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(state.PID), "stat")) + if err != nil { + return false + } + closeParen := strings.LastIndexByte(string(stat), ')') + if closeParen < 0 || len(stat) <= closeParen+2 || stat[closeParen+2] == 'Z' { + return false + } + executable, err := os.Readlink(filepath.Join("/proc", strconv.Itoa(state.PID), "exe")) + if err != nil { + return false + } + resolvedBinary, err := filepath.EvalSymlinks(binary) + if err != nil { + resolvedBinary = binary + } + executable = strings.TrimSuffix(executable, " (deleted)") + if filepath.Clean(executable) != filepath.Clean(resolvedBinary) { + return false + } + data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(state.PID), "cmdline")) + if err != nil || len(data) == 0 { + return false + } + arguments := strings.Split(strings.TrimRight(string(data), "\x00"), "\x00") + return commandHasOption(arguments, "--socket-path", state.SocketPath) && + commandHasOption(arguments, "--shared-dir", state.SharedDir) && + commandHasOption(arguments, "--sandbox", "namespace") && + commandHasOption(arguments, "--migration-mode", "find-paths") && + commandHasOption(arguments, "--migration-on-error", "abort") && + commandHasFlag(arguments, "--readonly") && + commandHasFlag(arguments, "--no-announce-submounts") && + commandHasFlag(arguments, "--inode-file-handles=never") +} + +func commandHasOption(arguments []string, option, value string) bool { + for index := 0; index+1 < len(arguments); index++ { + if arguments[index] == option && arguments[index+1] == value { + return true + } + } + return false +} + +func commandHasFlag(arguments []string, flag string) bool { + for _, argument := range arguments { + if argument == flag { + return true + } + } + return false +} + +func unmountFirecrackerVirtioFSShared(sharedDir string) error { + err := unix.Unmount(sharedDir, unix.MNT_DETACH) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOENT) { + return nil + } + if err != nil { + return fmt.Errorf("unmount virtio-fs shared directory %s: %w", sharedDir, err) + } + return nil +} + +func cleanupFirecrackerVirtioFS(state *firecrackerVirtioFSState, binary string) error { + if state == nil { + return nil + } + stopFirecrackerVirtioFS(state, binary) + unmountErr := unmountFirecrackerVirtioFSShared(state.SharedDir) + var removeErr error + if unmountErr == nil { + removeErr = os.RemoveAll(state.SharedDir) + } + return errors.Join( + unmountErr, + removeFirecrackerSocket(state.SocketPath), + removeErr, + ) +} + +func attachFirecrackerVirtioFSProcessGroup(cgroupPath string, leaderPID int) error { + // Move the leader first so any later fork inherits the destination cgroup. + // Then move children that virtiofsd forked before its socket became ready. + if err := attachFirecrackerProcess(cgroupPath, leaderPID); err != nil { + return fmt.Errorf("attach virtiofsd leader %d to cgroup: %w", leaderPID, err) + } + pids, err := firecrackerProcessGroupPIDs(leaderPID) + if err != nil { + return err + } + for _, pid := range pids { + if pid == leaderPID { + continue + } + if err := attachFirecrackerProcess(cgroupPath, pid); err != nil { + if errors.Is(err, syscall.ESRCH) { + continue + } + return fmt.Errorf("attach virtiofsd process %d to cgroup: %w", pid, err) + } + } + return nil +} + +func firecrackerProcessGroupPIDs(leaderPID int) ([]int, error) { + if leaderPID <= 1 { + return nil, errors.New("invalid virtiofsd process group leader") + } + groupID, err := syscall.Getpgid(leaderPID) + if err != nil { + return nil, fmt.Errorf("get virtiofsd process group: %w", err) + } + if groupID != leaderPID { + return nil, fmt.Errorf( + "virtiofsd pid %d is not its process group leader (group %d)", + leaderPID, + groupID, + ) + } + entries, err := os.ReadDir("/proc") + if err != nil { + return nil, fmt.Errorf("read process table: %w", err) + } + pids := make([]int, 0, 2) + leaderFound := false + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil || pid <= 1 { + continue + } + stat, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat")) + if err != nil { + continue + } + processGroup, err := firecrackerProcessGroupFromStat(stat) + if err != nil || processGroup != groupID { + continue + } + pids = append(pids, pid) + leaderFound = leaderFound || pid == leaderPID + } + if !leaderFound { + return nil, fmt.Errorf("virtiofsd process group leader %d disappeared", leaderPID) + } + sort.Ints(pids) + return pids, nil +} + +func firecrackerProcessGroupFromStat(stat []byte) (int, error) { + closeParen := strings.LastIndexByte(string(stat), ')') + if closeParen < 0 || len(stat) <= closeParen+2 { + return 0, errors.New("malformed process stat") + } + fields := strings.Fields(string(stat[closeParen+2:])) + if len(fields) < 3 { + return 0, errors.New("process stat lacks process group") + } + groupID, err := strconv.Atoi(fields[2]) + if err != nil || groupID <= 1 { + return 0, fmt.Errorf("invalid process group %q", fields[2]) + } + return groupID, nil +} diff --git a/pkg/runtime/firecracker/virtiofs_integration_test.go b/pkg/runtime/firecracker/virtiofs_integration_test.go new file mode 100644 index 0000000..8ac9678 --- /dev/null +++ b/pkg/runtime/firecracker/virtiofs_integration_test.go @@ -0,0 +1,66 @@ +//go:build linux && firecracker_integration + +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firecracker + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +func TestPrepareFirecrackerVirtioFSSharedReadOnly(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires mount privileges") + } + root := t.TempDir() + source := filepath.Join(root, "source") + shared := filepath.Join(root, "shared") + if err := os.Mkdir(source, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "data"), []byte("content"), 0644); err != nil { + t.Fatal(err) + } + state := &firecrackerVirtioFSState{ + SocketPath: filepath.Join(root, "virtiofs.sock"), + SharedDir: shared, + } + t.Cleanup(func() { + if err := cleanupFirecrackerVirtioFS(state, "/nonexistent/virtiofsd"); err != nil { + t.Errorf("cleanup virtio-fs staging: %v", err) + } + }) + if err := prepareFirecrackerVirtioFSShared(shared, []firecrackerVirtioFSExport{{ + Source: source, RelativePath: "rootfs", + }}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(shared, "rootfs", "data")) + if err != nil || string(data) != "content" { + t.Fatalf("staged data = %q, %v", data, err) + } + err = os.WriteFile(filepath.Join(shared, "rootfs", "blocked"), []byte("write"), 0644) + if !errors.Is(err, unix.EROFS) { + t.Fatalf("write through read-only staging mount = %v", err) + } + if err := os.WriteFile(filepath.Join(source, "host-write"), []byte("ok"), 0644); err != nil { + t.Fatalf("source unexpectedly became read-only: %v", err) + } +} diff --git a/pkg/runtime/firecracker/virtiofs_test.go b/pkg/runtime/firecracker/virtiofs_test.go new file mode 100644 index 0000000..68988c4 --- /dev/null +++ b/pkg/runtime/firecracker/virtiofs_test.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firecracker + +import ( + "fmt" + "os/exec" + "path/filepath" + "testing" +) + +func TestFirecrackerVirtioFSExportPath(t *testing.T) { + root := t.TempDir() + path, err := firecrackerVirtioFSExportPath(root, "mounts/0001") + if err != nil { + t.Fatal(err) + } + if path != filepath.Join(root, "mounts/0001") { + t.Fatalf("export path = %q", path) + } + for _, invalid := range []string{"", ".", "..", "../escape", "/absolute"} { + if _, err := firecrackerVirtioFSExportPath(root, invalid); err == nil { + t.Fatalf("accepted export path %q", invalid) + } + } +} + +func TestCommandHasOption(t *testing.T) { + arguments := []string{ + "virtiofsd", + "--shared-dir", "/storage/shared", + "--socket-path", "/run/virtiofs.sock", + "--readonly", + "--no-announce-submounts", + } + if !commandHasOption(arguments, "--shared-dir", "/storage/shared") { + t.Fatal("shared-dir option was not found") + } + if commandHasOption(arguments, "--socket-path", "/run/other.sock") { + t.Fatal("mismatched socket-path option was accepted") + } + if commandHasOption(arguments, "--readonly", "--sandbox") { + t.Fatal("flag without a value was accepted as an option pair") + } + if !commandHasFlag(arguments, "--readonly") || + !commandHasFlag(arguments, "--no-announce-submounts") || + commandHasFlag(arguments, "--inode-file-handles=never") { + t.Fatalf("flag matching failed for %q", arguments) + } +} + +func TestFirecrackerProcessGroupFromStat(t *testing.T) { + for _, test := range []struct { + name string + stat string + want int + }{ + { + name: "simple command", + stat: "123 (virtiofsd) S 1 123 123 0 -1", + want: 123, + }, + { + name: "command with parentheses", + stat: "456 (virtiofsd (worker)) S 123 456 456 0 -1", + want: 456, + }, + } { + t.Run(test.name, func(t *testing.T) { + got, err := firecrackerProcessGroupFromStat([]byte(test.stat)) + if err != nil || got != test.want { + t.Fatalf("process group = %d, %v; want %d", got, err, test.want) + } + }) + } + for index, stat := range []string{ + "", + "123 virtiofsd S 1 123", + "123 (virtiofsd) S 1", + "123 (virtiofsd) S 1 invalid", + "123 (virtiofsd) S 1 1", + } { + t.Run(fmt.Sprintf("invalid-%d", index), func(t *testing.T) { + if _, err := firecrackerProcessGroupFromStat([]byte(stat)); err == nil { + t.Fatalf("accepted malformed stat %q", stat) + } + }) + } +} + +func TestWaitFirecrackerVirtioFSCommandCanBeObservedMoreThanOnce(t *testing.T) { + command := exec.Command("/bin/sh", "-c", "exit 23") + if err := command.Start(); err != nil { + t.Fatal(err) + } + process := waitFirecrackerVirtioFSCommand(command) + for index := 0; index < 2; index++ { + err := process.wait() + exitErr, ok := err.(*exec.ExitError) + if !ok || exitErr.ExitCode() != 23 { + t.Fatalf("wait %d error = %v, want exit status 23", index, err) + } + } +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 0468115..ea70d93 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -28,12 +28,7 @@ The flow: `ktap` lifecycle; 11. checkpoints the same runsc or Firecracker sandbox ten consecutive times, verifies it keeps running, and restores the tenth artifact; -12. verifies Firecracker's EROFS root and mount contract, OCI-to-EROFS rootfs - conversion, private ext4 overlay, native writable ext4 mounts (including - checkpoint/restore), quota exhaustion, guest exec/TTY protocol, direct - service access, local DNAT, network ACL and managed DNS replacement, crash - recovery, stale-policy removal, exit-code recovery when the daemon is - unavailable, and reuse of the same TAP without policy leakage; and +12. verifies Firecracker's EROFS and virtio-fs root and mount contracts, OCI/Nydus directory rootfs, private ext4 overlay, native writable ext4 mounts (including checkpoint/restore), quota exhaustion, guest exec/TTY protocol, direct service access, local DNAT, network ACL and managed DNS replacement, crash recovery, stale-policy removal, exit-code recovery when the daemon is unavailable, and reuse of the same TAP without policy leakage; and 13. runs concurrent Redis SET/GET traffic from every runtime to a sibling Redis container through SNAT and from that container to the sandbox's published port through DNAT. The test verifies the translated source @@ -119,27 +114,43 @@ FIRECRACKER_INITRD=/opt/firecracker/initrd.img \ make e2e ``` +Validate an unpublished migration-capable Firecracker and virtiofsd together +without changing `third_party/runtime-versions.env`: + +```bash +TMPDIR=/xfs/build-tmp make firecracker-initrd + +E2E_RUNTIME=firecracker \ +E2E_FIRECRACKER_VIRTIOFS=1 \ +E2E_HOME_FIXTURE_PARENT=/xfs/test-tmp \ +E2E_KEEP_HOME_FIXTURE=1 \ +RUN_UNIT_TESTS=0 \ +FIRECRACKER_BINARY=/path/to/candidate/firecracker \ +FIRECRACKER_KERNEL=/path/to/candidate/vmlinux \ +FIRECRACKER_INITRD="${PWD}/output/initrd.img" \ +FIRECRACKER_VIRTIOFSD=/path/to/virtiofsd \ +make e2e +``` + +The virtio-fs mode uses a directory rootfs and a read-only directory mount in +the main lifecycle and checkpoint/restore paths. It also checks the +`virtiofs.state` sidecar and compatibility digest. Point +`E2E_HOME_FIXTURE_PARENT` at XFS to exercise reflinked checkpoint memory and +filestore data there. The keep flag retains that fixture and prints its exact +path; the default remains automatic cleanup. + `KATA_ROOT` must contain the runtime-rs shim, Dragonball configuration, guest kernel, and guest image at their upstream archive paths. The sandbox logger is built with sandboxd. The Firecracker kernel must provide the facilities listed in [the runtime guide](../../doc/runtime.md), and the initrd must contain the matching `firecracker-agent` as `/init`. -Set `RUN_UNIT_TESTS=0` to skip unit tests while rerunning a privileged -scenario. Set `E2E_STRESS_ROUNDS` to a positive number and -`E2E_STRESS_CONCURRENCY` to 1 through 8 to run concurrent lifecycle rounds. -Targeted runtime cases enable the Redis network soak by default. Set -`E2E_NETWORK_SOAK=1` with one selected `E2E_RUNTIME` to enable it for a -direct `make e2e` invocation. The harness uses a digest-pinned Redis image; -`E2E_REDIS_IMAGE` can point at a preloaded equivalent when Docker Hub is not -reachable. -`E2E_RUNTIME=all` means runsc plus runc; Kata and Firecracker stay explicit -for targeted images. `E2E_SKIP_BUILD=1` reuses -`SANDBOXD_E2E_IMAGE`, and `E2E_RUN_CGROUP_DISABLED=0` suppresses the second -runsc cgroup-disabled container. These controls are used by the runtime-case -wrapper. +Set `RUN_UNIT_TESTS=0` to skip unit tests while rerunning a privileged scenario. Set `E2E_STRESS_ROUNDS` to a positive number and `E2E_STRESS_CONCURRENCY` to 1 through 32 to run concurrent lifecycle rounds. Targeted runtime cases enable the Redis network soak by default. Set `E2E_NETWORK_SOAK=1` with one selected `E2E_RUNTIME` to enable it for a direct `make e2e` invocation. The harness uses a digest-pinned Redis image; `E2E_REDIS_IMAGE` can point at a preloaded equivalent when Docker Hub is not reachable. +`E2E_RUNTIME=all` means runsc plus runc; Kata and Firecracker stay explicit for targeted images. `E2E_SKIP_BUILD=1` reuses `SANDBOXD_E2E_IMAGE`, and `E2E_RUN_CGROUP_DISABLED=0` suppresses the second runsc cgroup-disabled container. These controls are used by the runtime-case wrapper. `E2E_RUNC_ONLY=1` remains a deprecated alias for `E2E_RUNTIME=runc`. +For a Firecracker virtio-fs storage soak, set `E2E_STRESS_ROOTFS_HOST` to a host directory that contains executable `/bin/sh`, `/stress-data/large.bin`, `/stress-data/small.master`, and a populated `/stress-data/small` directory. The harness mounts it read-only into the E2E container, continuously verifies and scans it from every concurrent guest, and keeps each guest's writes in its private layer. `E2E_STRESS_CHECKPOINT=1` checkpoints and restores one guest in every round while the remaining guests continue their read workload. The host directory can be a distill-fs Nydus FUSE mount to exercise the complete Nydus-to-virtio-fs data path. Set `E2E_STRESS_ONLY=1` to skip the ordinary runtime cases when iterating on a soak; it requires Firecracker virtio-fs and at least one stress round. + ## Host requirements - an accessible Docker daemon; diff --git a/test/e2e/checkpoint-restore/main.go b/test/e2e/checkpoint-restore/main.go index e0bf81a..7e3f290 100644 --- a/test/e2e/checkpoint-restore/main.go +++ b/test/e2e/checkpoint-restore/main.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "os" + "strings" "time" runtime "github.com/inclusionAI/sandboxd/api/runtime/v1" @@ -48,6 +49,18 @@ type options struct { leaveRunning bool snapshotType string workloadCmd string + mounts stringList +} + +type stringList []string + +func (values *stringList) String() string { + return strings.Join(*values, ",") +} + +func (values *stringList) Set(value string) error { + *values = append(*values, value) + return nil } func main() { @@ -78,6 +91,8 @@ func main() { "checkpoint flavor: empty (auto), Full, Incremental, or SoftDirty") flag.StringVar(&value.workloadCmd, "workload-cmd", "", "override the built-in start workload command (template warmup hooks)") + flag.Var(&value.mounts, "mount", + "repeatable mount formatted as host_path:target[:type[:opt1,opt2]]") flag.Parse() if err := run(value); err != nil { @@ -135,6 +150,10 @@ func start( if value.storageMB > ^uint64(0)/(1024*1024) { return errors.New("--storage-mb overflows bytes") } + mounts, err := parseMountFlags(value.mounts) + if err != nil { + return err + } request := &runtime.StartRequest{ SandboxID: value.sandboxID, Runtime: value.runtime, @@ -151,6 +170,7 @@ func start( }, Cwd: "/", Network: "sandbox", + Mounts: mounts, Stdout: "/var/log/sandboxd/checkpoint-workload.stdout", Stderr: "/var/log/sandboxd/checkpoint-runtime.stderr", Resources: map[string]float64{ @@ -182,6 +202,36 @@ func start( return nil } +func parseMountFlags(values []string) ([]*runtime.Mount, error) { + mounts := make([]*runtime.Mount, 0, len(values)) + for _, value := range values { + parts := strings.SplitN(value, ":", 4) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return nil, fmt.Errorf( + "invalid mount %q, expected host_path:target[:type[:opt1,opt2]]", + value, + ) + } + mountType := "bind" + if len(parts) >= 3 && parts[2] != "" { + mountType = parts[2] + } + options := []string{"rbind", "rw"} + if len(parts) == 4 && parts[3] != "" { + options = strings.Split(parts[3], ",") + } + mounts = append(mounts, &runtime.Mount{ + Type: mountType, + Target: parts[1], + Options: options, + Source: &runtime.Mount_HostPath{ + HostPath: parts[0], + }, + }) + } + return mounts, nil +} + // workloadCommand returns the guest workload for a start: an explicit hook // overrides the built-in regression workload. Hook authors keep full control // but must keep the sandbox alive — end with an idle loop. diff --git a/test/e2e/checkpoint-restore/main_test.go b/test/e2e/checkpoint-restore/main_test.go new file mode 100644 index 0000000..a2d3e47 --- /dev/null +++ b/test/e2e/checkpoint-restore/main_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" +) + +func TestParseMountFlags(t *testing.T) { + mounts, err := parseMountFlags([]string{ + "/host/data:/mnt/data:bind:ro,nodev", + "tmpfs:/run/cache:tmpfs:rw,size=1m", + }) + if err != nil { + t.Fatal(err) + } + if len(mounts) != 2 { + t.Fatalf("mount count = %d", len(mounts)) + } + if mounts[0].GetHostPath() != "/host/data" || + mounts[0].GetTarget() != "/mnt/data" || + mounts[0].GetType() != "bind" || + !reflect.DeepEqual(mounts[0].GetOptions(), []string{"ro", "nodev"}) { + t.Fatalf("first mount = %+v", mounts[0]) + } + if mounts[1].GetHostPath() != "tmpfs" || + mounts[1].GetType() != "tmpfs" || + !reflect.DeepEqual(mounts[1].GetOptions(), []string{"rw", "size=1m"}) { + t.Fatalf("second mount = %+v", mounts[1]) + } +} + +func TestParseMountFlagsRejectsMalformedValue(t *testing.T) { + for _, value := range []string{"", "source", ":/target", "source:"} { + if _, err := parseMountFlags([]string{value}); err == nil { + t.Fatalf("accepted malformed mount %q", value) + } + } +} diff --git a/test/e2e/e2e-run.sh b/test/e2e/e2e-run.sh index 905c464..388f4a9 100755 --- a/test/e2e/e2e-run.sh +++ b/test/e2e/e2e-run.sh @@ -29,6 +29,7 @@ EROFS_MOUNT_ROOT="${E2E_EROFS_MOUNT_ROOT:-/e2e/erofs-mount-root}" EROFS_MOUNT_IMAGE="${E2E_EROFS_MOUNT_IMAGE:-/e2e/data.erofs}" FIRECRACKER_KERNEL="${E2E_FIRECRACKER_KERNEL:-/opt/firecracker/vmlinux}" FIRECRACKER_CHECKPOINT_MODE="${E2E_FIRECRACKER_CHECKPOINT_MODE:-}" +FIRECRACKER_VIRTIOFS="${E2E_FIRECRACKER_VIRTIOFS:-0}" FIRECRACKER_INITRD="${E2E_FIRECRACKER_INITRD:-/opt/firecracker/initrd.img}" OCI_ROOTFS_IMAGE="${E2E_OCI_ROOTFS_IMAGE:-docker.io/library/redis:7-alpine}" FIRECRACKER_OVERLAY_BYTES="${E2E_FIRECRACKER_OVERLAY_BYTES:-134217728}" @@ -52,6 +53,9 @@ REDIS_RESULT_KEY="${E2E_REDIS_RESULT_KEY:-}" REDIS_BENCHMARK_REQUESTS="${E2E_REDIS_BENCHMARK_REQUESTS:-20000}" STRESS_ROUNDS="${E2E_STRESS_ROUNDS:-0}" STRESS_CONCURRENCY="${E2E_STRESS_CONCURRENCY:-8}" +STRESS_ROOTFS="${E2E_STRESS_ROOTFS:-}" +STRESS_CHECKPOINT="${E2E_STRESS_CHECKPOINT:-0}" +STRESS_ONLY="${E2E_STRESS_ONLY:-0}" DISABLE_CGROUP="${E2E_DISABLE_CGROUP:-0}" CPU_LIMIT_MODE="${E2E_CPU_LIMIT_MODE:-quota}" E2E_RUNTIME="${E2E_RUNTIME:-all}" @@ -187,9 +191,16 @@ preflight() { assert_sandboxd_home_is_disk_backed [ "$(id -u)" = "0" ] || fail "e2e container must run as root" [[ "${STRESS_ROUNDS}" =~ ^[0-9]+$ ]] || fail "E2E_STRESS_ROUNDS must be a non-negative integer" - [[ "${STRESS_CONCURRENCY}" =~ ^[1-8]$ ]] || fail "E2E_STRESS_CONCURRENCY must be between 1 and 8" + [[ "${STRESS_CONCURRENCY}" =~ ^([1-9]|[12][0-9]|3[0-2])$ ]] || + fail "E2E_STRESS_CONCURRENCY must be between 1 and 32" + [[ "${STRESS_CHECKPOINT}" =~ ^[01]$ ]] || + fail "E2E_STRESS_CHECKPOINT must be 0 or 1" + [[ "${STRESS_ONLY}" =~ ^[01]$ ]] || + fail "E2E_STRESS_ONLY must be 0 or 1" [[ "${DISABLE_CGROUP}" =~ ^[01]$ ]] || fail "E2E_DISABLE_CGROUP must be 0 or 1" [[ "${NETWORK_SOAK}" =~ ^[01]$ ]] || fail "E2E_NETWORK_SOAK must be 0 or 1" + [[ "${FIRECRACKER_VIRTIOFS}" =~ ^[01]$ ]] || + fail "E2E_FIRECRACKER_VIRTIOFS must be 0 or 1" [[ "${REDIS_BENCHMARK_REQUESTS}" =~ ^[1-9][0-9]*$ ]] || fail "E2E_REDIS_BENCHMARK_REQUESTS must be positive" [[ "${CPU_LIMIT_MODE}" =~ ^(shares|quota)$ ]] || fail "E2E_CPU_LIMIT_MODE must be shares or quota" case "${E2E_RUNTIME}" in @@ -216,6 +227,36 @@ preflight() { if [ "${E2E_RUNTIME}" = "firecracker" ] && [ "${DISABLE_CGROUP}" = "1" ]; then fail "Firecracker e2e requires sandbox-managed cgroups" fi + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ] && + [ "${E2E_RUNTIME}" != "firecracker" ]; then + fail "E2E_FIRECRACKER_VIRTIOFS requires E2E_RUNTIME=firecracker" + fi + if [ -n "${STRESS_ROOTFS}" ]; then + [ "${E2E_RUNTIME}" = "firecracker" ] && + [ "${FIRECRACKER_VIRTIOFS}" = "1" ] || + fail "E2E_STRESS_ROOTFS requires Firecracker virtio-fs" + [ -x "${STRESS_ROOTFS}/bin/sh" ] || + fail "E2E_STRESS_ROOTFS lacks executable /bin/sh" + [ -f "${STRESS_ROOTFS}/stress-data/large.bin" ] || + fail "E2E_STRESS_ROOTFS lacks /stress-data/large.bin" + [ -f "${STRESS_ROOTFS}/stress-data/small.master" ] || + fail "E2E_STRESS_ROOTFS lacks /stress-data/small.master" + [ -d "${STRESS_ROOTFS}/stress-data/small" ] || + fail "E2E_STRESS_ROOTFS lacks /stress-data/small" + fi + if [ "${STRESS_CHECKPOINT}" = "1" ] && { + [ "${E2E_RUNTIME}" != "firecracker" ] || + [ "${FIRECRACKER_VIRTIOFS}" != "1" ]; + }; then + fail "E2E_STRESS_CHECKPOINT requires Firecracker virtio-fs" + fi + if [ "${STRESS_ONLY}" = "1" ] && { + [ "${E2E_RUNTIME}" != "firecracker" ] || + [ "${FIRECRACKER_VIRTIOFS}" != "1" ] || + [ "${STRESS_ROUNDS}" = "0" ]; + }; then + fail "E2E_STRESS_ONLY requires Firecracker virtio-fs stress rounds" + fi if [ "${NETWORK_SOAK}" = "1" ]; then [ "${E2E_RUNTIME}" != "all" ] || fail "network soak requires one selected runtime" @@ -256,6 +297,10 @@ preflight() { firecracker) command -v firecracker >/dev/null 2>&1 || fail "missing command: firecracker" command -v mkfs.ext4 >/dev/null 2>&1 || fail "missing command: mkfs.ext4" + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + command -v virtiofsd >/dev/null 2>&1 || + fail "missing command: virtiofsd" + fi [ -c /dev/kvm ] || fail "Firecracker e2e requires /dev/kvm" [ -f "${FIRECRACKER_KERNEL}" ] || fail "missing Firecracker kernel" [ -f "${FIRECRACKER_INITRD}" ] || fail "missing Firecracker initrd" @@ -357,6 +402,18 @@ EOF local runtime_binaries local node_resource_config="" + local max_instance_num=8 + local interface_cache_size=1 + if [ "${STRESS_CONCURRENCY}" -gt "${max_instance_num}" ]; then + max_instance_num="${STRESS_CONCURRENCY}" + fi + if [ "${STRESS_ROUNDS}" -gt 0 ]; then + # Keep the full stress working set reusable. The interface manager trims + # idle entries every 30 seconds; a cache of one can otherwise destroy + # endpoints between back-to-back rounds and make allocation fail before + # the storage path is exercised. + interface_cache_size="${STRESS_CONCURRENCY}" + fi case "${E2E_RUNTIME}" in all) runtime_binaries=$'runsc = "/usr/local/bin/runsc"\nrunc = "/usr/local/bin/runc"' @@ -383,6 +440,10 @@ EOF e2e_fc_checkpoint_mode_cfg="$(printf 'checkpoint_mode = "%s"' \ "${FIRECRACKER_CHECKPOINT_MODE}")" fi + local e2e_fc_virtiofs_cfg="" + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + e2e_fc_virtiofs_cfg=$'virtiofs_enabled = true\nvirtiofsd_path = "/usr/local/bin/virtiofsd"' + fi cat > "${CONFIG_FILE}" < /mnt/host/checkpoint-write' \ + >/tmp/firecracker-checkpoint-mount-write.log 2>&1; then + cat /tmp/firecracker-checkpoint-mount-write.log >&2 + fail "${suffix} checkpoint source virtio-fs mount was writable" + fi + fi sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ 'echo checkpoint-state-ok > /var/checkpoint-persist' if [ "${runtime}" = "firecracker" ]; then @@ -827,6 +907,18 @@ run_checkpoint_restore_check() { assert_snapshot_type "${checkpoint_dir}" "Full" \ "${suffix} baseline checkpoint ${checkpoint_index}" fi + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + [ -s "${checkpoint_dir}/virtiofs.state" ] || + fail "${suffix} checkpoint ${checkpoint_index} lacks virtiofs.state" + jq -e ' + .virtio_fs == true and + (.digests["virtiofs.state"] | + test("^[0-9a-f]{64}$")) and + (.compat.virtiofsd | + test("^[0-9a-f]{64}$")) + ' "${checkpoint_dir}/manifest.json" >/dev/null || + fail "${suffix} checkpoint ${checkpoint_index} lacks virtio-fs metadata" + fi fi source_after="" @@ -869,6 +961,20 @@ run_checkpoint_restore_check() { local persisted persisted="$(sbox_cmd exec "${SANDBOX_ID}" /bin/cat /var/checkpoint-persist)" assert_eq "${persisted}" "checkpoint-state-ok" "${suffix} restored writable state" + if [ "${runtime}" = "firecracker" ] && + [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + local restored_mount + restored_mount="$(sbox_cmd exec "${SANDBOX_ID}" \ + /bin/cat /mnt/host/input.txt)" + assert_eq "${restored_mount}" "host-mount-ok" \ + "${suffix} restored virtio-fs mount" + if sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ + 'echo unexpected > /mnt/host/restored-write' \ + >/tmp/firecracker-restored-mount-write.log 2>&1; then + cat /tmp/firecracker-restored-mount-write.log >&2 + fail "${suffix} restored virtio-fs mount was writable" + fi + fi if [ "${runtime}" = "firecracker" ]; then local restored_init restored_init="$(sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ @@ -1054,7 +1160,8 @@ run_firecracker_post_restore_chain() { run_network_soak() { local runtime="${1}" local rootfs="${REDIS_ROOTFS}" - if [ "${runtime}" = "firecracker" ]; then + if [ "${runtime}" = "firecracker" ] && + [ "${FIRECRACKER_VIRTIOFS}" != "1" ]; then rootfs="${REDIS_EROFS_ROOTFS}" fi @@ -1263,6 +1370,24 @@ wait_for_cgroup_count() { fail "cgroup child count did not reach ${expected}; last count: ${count}" } +assert_virtiofsd_cgroups() { + local comm_path + local cgroup_path + local pid + local count=0 + for comm_path in /proc/[0-9]*/comm; do + [ "$(cat "${comm_path}" 2>/dev/null || true)" = "virtiofsd" ] || continue + cgroup_path="${comm_path%/comm}/cgroup" + pid="${comm_path%/comm}" + pid="${pid##*/}" + grep -Eq "^[^:]*:[^:]*:/${CGROUP_ROOT}/" "${cgroup_path}" || + fail "virtiofsd pid ${pid} escaped ${CGROUP_ROOT} cgroups" + count=$((count + 1)) + done + [ "${count}" -ge "${STRESS_CONCURRENCY}" ] || + fail "found only ${count} virtiofsd processes for ${STRESS_CONCURRENCY} sandboxes" +} + wait_for_process_exit() { local pid="$1" local description="$2" @@ -1368,10 +1493,32 @@ run_stress_checks() { return fi + local memory_mb=128 + local cpu_millicores=100 + if [ "${runtime}" = "firecracker" ]; then + memory_mb=256 + fi + local marker="" + local workload="/bin/sleep 300" + if [ -n "${STRESS_ROOTFS}" ]; then + rootfs="${STRESS_ROOTFS}" + cpu_millicores=1000 + local large_sha + local small_sha + local small_count + large_sha="$(sha256sum "${rootfs}/stress-data/large.bin" | awk '{print $1}')" + small_sha="$(sha256sum "${rootfs}/stress-data/small.master" | awk '{print $1}')" + small_count="$(find "${rootfs}/stress-data/small" -type f | wc -l)" + marker="${large_sha}:${small_sha}:${small_count}" + workload='while :; do large="$(sha256sum /stress-data/large.bin)"; large="${large%% *}"; small="$(sha256sum /stress-data/small.master)"; small="${small%% *}"; find /stress-data/small -type f -exec cat {} + >/dev/null; count="$(find /stress-data/small -type f | wc -l)"; printf "%s:%s:%s\n" "$large" "$small" "$count" > /var/virtiofs-stress; done' + fi + log "running ${STRESS_ROUNDS} ${runtime} stress rounds at concurrency ${STRESS_CONCURRENCY}" local round local slot local id + local request_file + local checkpoint_dir local -a pids for round in $(seq 1 "${STRESS_ROUNDS}"); do STRESS_IDS=() @@ -1379,19 +1526,40 @@ run_stress_checks() { for slot in $(seq 1 "${STRESS_CONCURRENCY}"); do id="sbox-e2e-stress-${round}-${slot}" STRESS_IDS+=("${id}") - sbox_cmd start \ - --quiet \ - --runtime "${runtime}" \ - --sandbox-id "${id}" \ - --rootfs "${rootfs}" \ - --cpu-millicores 100 \ - --memory-mb 128 \ - /bin/sleep 300 >"/tmp/${id}.start.log" 2>&1 & + if [ "${STRESS_CHECKPOINT}" = "1" ] && [ "${slot}" = "1" ]; then + request_file="${SANDBOXD_HOME}/stress-${round}.request.json" + checkpoint-restore \ + --action start \ + --socket "${SOCKET}" \ + --runtime "${runtime}" \ + --rootfs "${rootfs}" \ + --sandbox-id "${id}" \ + --request-file "${request_file}" \ + --cpu "${cpu_millicores}" \ + --memory-mb "${memory_mb}" \ + --storage-mb 64 \ + --workload-cmd "${workload}" \ + >"/tmp/${id}.start.log" 2>&1 & + else + sbox_cmd start \ + --quiet \ + --runtime "${runtime}" \ + --sandbox-id "${id}" \ + --rootfs "${rootfs}" \ + --cpu-millicores "${cpu_millicores}" \ + --memory-mb "${memory_mb}" \ + /bin/sh -c "${workload}" \ + >"/tmp/${id}.start.log" 2>&1 & + fi pids+=("$!") done for slot in "${!pids[@]}"; do if ! wait "${pids[$slot]}"; then cat "/tmp/${STRESS_IDS[$slot]}.start.log" >&2 + if [ -f "${LOG_FILE}" ]; then + log "sandboxd log at failed stress start" + tail -300 "${LOG_FILE}" >&2 + fi fail "stress start failed for ${STRESS_IDS[$slot]}" fi done @@ -1399,6 +1567,62 @@ run_stress_checks() { wait_for_state "${id}" "SANDBOX_STATE_RUNNING" done wait_for_cgroup_count "${STRESS_CONCURRENCY}" + if [ "${runtime}" = "firecracker" ] && [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + assert_virtiofsd_cgroups + fi + + if [ -n "${marker}" ]; then + for id in "${STRESS_IDS[@]}"; do + local got="" + local attempt + for attempt in $(seq 1 1200); do + got="$(sbox_cmd exec "${id}" /bin/cat \ + /var/virtiofs-stress 2>/dev/null || true)" + [ "${got}" = "${marker}" ] && break + sleep 0.1 + done + [ "${got}" = "${marker}" ] || + fail "stress read verification failed for ${id}: ${got@Q}" + done + fi + + if [ "${STRESS_CHECKPOINT}" = "1" ]; then + local source_id="${STRESS_IDS[0]}" + local restored_id="${source_id}-restored" + checkpoint_dir="${SANDBOXD_HOME}/stress-${round}.checkpoint" + checkpoint-restore \ + --action checkpoint \ + --socket "${SOCKET}" \ + --sandbox-id "${source_id}" \ + --checkpoint-dir "${checkpoint_dir}" \ + --checkpoint-timeout-seconds 180 \ + --compress=true \ + --leave-running=true + [ -s "${checkpoint_dir}/virtiofs.state" ] || + fail "stress checkpoint lacks virtiofs.state" + sbox_cmd delete "${source_id}" + if ! checkpoint-restore \ + --action restore \ + --timeout 60s \ + --socket "${SOCKET}" \ + --target-id "${restored_id}" \ + --request-file "${request_file}" \ + --checkpoint-dir "${checkpoint_dir}" >/dev/null; then + if [ -f "${LOG_FILE}" ]; then + log "sandboxd log at failed stress restore" + tail -300 "${LOG_FILE}" >&2 + fi + fail "stress restore failed for ${restored_id}" + fi + STRESS_IDS[0]="${restored_id}" + wait_for_state "${restored_id}" "SANDBOX_STATE_RUNNING" 300 + if [ -n "${marker}" ]; then + wait_for_exec_output "${restored_id}" "${marker}" \ + /bin/cat /var/virtiofs-stress + fi + wait_for_cgroup_count "${STRESS_CONCURRENCY}" + assert_virtiofsd_cgroups + fi pids=() for id in "${STRESS_IDS[@]}"; do @@ -1412,6 +1636,10 @@ run_stress_checks() { fi done wait_for_cgroup_count 1 + if [ "${STRESS_CHECKPOINT}" = "1" ]; then + rm -rf -- "${checkpoint_dir}" + rm -f -- "${request_file}" + fi STRESS_IDS=() done log "stress checks passed" @@ -1633,7 +1861,15 @@ run_kata_checks() { } run_firecracker_checks() { - log "testing Firecracker EROFS root, writable layer, exec, and network" + local rootfs="${EROFS_ROOTFS}" + local host_mount="${HOST_MOUNT}/input.txt:/mnt/host/input.txt:bind:ro" + local root_description="EROFS" + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + rootfs="${ROOTFS}" + host_mount="${HOST_MOUNT}:/mnt/host:bind:ro" + root_description="virtio-fs directory" + fi + log "testing Firecracker ${root_description} root, writable layer, exec, and network" local main_stdout="/tmp/firecracker-main.stdout" local main_stderr="/tmp/firecracker-main.stderr" rm -f "${main_stdout}" "${main_stderr}" /tmp/firecracker-exec.stderr @@ -1642,10 +1878,10 @@ run_firecracker_checks() { --quiet \ --runtime firecracker \ --sandbox-id sbox-e2e-firecracker \ - --rootfs "${EROFS_ROOTFS}" \ + --rootfs "${rootfs}" \ --cwd / \ --env E2E_MARKER=firecracker-env-ok \ - --mount "${HOST_MOUNT}/input.txt:/mnt/host/input.txt:bind:ro" \ + --mount "${host_mount}" \ --mount "${EROFS_MOUNT_IMAGE}:/mnt/erofs:erofs:ro" \ --mount "tmpfs:/mnt/ram:tmpfs:rw,nosuid,nodev,noexec,size=1m,mode=0755" \ --extra-config \ @@ -1697,6 +1933,16 @@ run_firecracker_checks() { fail "Firecracker read-only injected file was writable" fi assert_eq "$(cat "${HOST_MOUNT}/input.txt")" "host-mount-ok" "Firecracker host file unchanged" + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + if sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ + 'echo changed > /mnt/host/new-file' \ + >/tmp/firecracker-directory-write.log 2>&1; then + cat /tmp/firecracker-directory-write.log >&2 + fail "Firecracker read-only directory mount accepted a new file" + fi + [ ! -e "${HOST_MOUNT}/new-file" ] || + fail "Firecracker directory mount write escaped to the host" + fi got="$(sbox_cmd exec "${SANDBOX_ID}" /bin/cat /mnt/erofs/input.txt)" assert_eq "${got}" "erofs-mount-ok" "Firecracker EROFS mount" @@ -1760,60 +2006,64 @@ run_firecracker_checks() { fail "recycled Firecracker TAP ${cached_tap} remained administratively up" fi - log "testing Firecracker rejects directory rootfs and mounts" - local rejected_id - if rejected_id="$(sbox_cmd start \ - --quiet \ - --runtime firecracker \ - --sandbox-id sbox-e2e-firecracker-directory-root \ - --rootfs "${ROOTFS}" \ - --cpu-millicores 100 \ - --memory-mb 256 \ - /bin/true 2>/tmp/firecracker-directory-root.log)"; then - sbox_cmd delete "${rejected_id}" || true - fail "Firecracker accepted a directory rootfs" - fi - if rejected_id="$(sbox_cmd start \ - --quiet \ - --runtime firecracker \ - --sandbox-id sbox-e2e-firecracker-directory-mount \ - --rootfs "${EROFS_ROOTFS}" \ - --mount "${EROFS_MOUNT_ROOT}:/mnt/dir:bind:ro" \ - --cpu-millicores 100 \ - --memory-mb 256 \ - /bin/true 2>/tmp/firecracker-directory-mount.log)"; then - sbox_cmd delete "${rejected_id}" || true - fail "Firecracker accepted a directory mount" + if [ "${FIRECRACKER_VIRTIOFS}" != "1" ]; then + log "testing Firecracker rejects directory rootfs and mounts" + local rejected_id + if rejected_id="$(sbox_cmd start \ + --quiet \ + --runtime firecracker \ + --sandbox-id sbox-e2e-firecracker-directory-root \ + --rootfs "${ROOTFS}" \ + --cpu-millicores 100 \ + --memory-mb 256 \ + /bin/true 2>/tmp/firecracker-directory-root.log)"; then + sbox_cmd delete "${rejected_id}" || true + fail "Firecracker accepted a directory rootfs" + fi + if rejected_id="$(sbox_cmd start \ + --quiet \ + --runtime firecracker \ + --sandbox-id sbox-e2e-firecracker-directory-mount \ + --rootfs "${EROFS_ROOTFS}" \ + --mount "${EROFS_MOUNT_ROOT}:/mnt/dir:bind:ro" \ + --cpu-millicores 100 \ + --memory-mb 256 \ + /bin/true 2>/tmp/firecracker-directory-mount.log)"; then + sbox_cmd delete "${rejected_id}" || true + fail "Firecracker accepted a directory mount" + fi fi - log "testing Firecracker OCI rootfs conversion" - local oci_root_id="sbox-e2e-firecracker-oci-root" - SANDBOX_ID="$(sbox_cmd start \ - --quiet \ - --runtime firecracker \ - --sandbox-id "${oci_root_id}" \ - --image-url "${OCI_ROOTFS_IMAGE}" \ - --cpu-millicores 100 \ - --memory-mb 256 \ - /bin/sh -c 'echo firecracker-oci-ready > /var/oci-rootfs; sleep 300')" - wait_for_state "${SANDBOX_ID}" "SANDBOX_STATE_RUNNING" - wait_for_exec_output "${SANDBOX_ID}" "firecracker-oci-ready" \ - /bin/cat /var/oci-rootfs - local redis_version - redis_version="$(sbox_cmd exec "${SANDBOX_ID}" redis-server --version)" - [[ "${redis_version}" == *"Redis server v="* ]] || \ - fail "Firecracker OCI rootfs did not preserve image content: ${redis_version@Q}" - sbox_cmd delete "${SANDBOX_ID}" - SANDBOX_ID="" + if [ "${FIRECRACKER_VIRTIOFS}" = "1" ]; then + log "testing Firecracker OCI/Nydus directory rootfs through virtio-fs" + local oci_root_id="sbox-e2e-firecracker-oci-root" + SANDBOX_ID="$(sbox_cmd start \ + --quiet \ + --runtime firecracker \ + --sandbox-id "${oci_root_id}" \ + --image-url "${OCI_ROOTFS_IMAGE}" \ + --cpu-millicores 100 \ + --memory-mb 256 \ + /bin/sh -c 'echo firecracker-oci-ready > /var/oci-rootfs; sleep 300')" + wait_for_state "${SANDBOX_ID}" "SANDBOX_STATE_RUNNING" + wait_for_exec_output "${SANDBOX_ID}" "firecracker-oci-ready" \ + /bin/cat /var/oci-rootfs + local redis_version + redis_version="$(sbox_cmd exec "${SANDBOX_ID}" redis-server --version)" + [[ "${redis_version}" == *"Redis server v="* ]] || \ + fail "Firecracker OCI/Nydus rootfs did not preserve image content: ${redis_version@Q}" + sbox_cmd delete "${SANDBOX_ID}" + SANDBOX_ID="" + fi local cached_taps_before cached_taps_before="$(list_cached_taps)" - log "testing Firecracker read-only EROFS root" + log "testing Firecracker read-only ${root_description} root" SANDBOX_ID="$(sbox_cmd start \ --quiet \ --runtime firecracker \ --sandbox-id sbox-e2e-firecracker-readonly \ - --rootfs "${EROFS_ROOTFS}" \ + --rootfs "${rootfs}" \ --rootfs-readonly \ --mount "${EROFS_MOUNT_IMAGE}:/mnt/erofs-readonly:erofs:ro" \ --cpu-millicores 100 \ @@ -1859,7 +2109,7 @@ run_firecracker_checks() { --quiet \ --runtime firecracker \ --sandbox-id sbox-e2e-firecracker-exit \ - --rootfs "${EROFS_ROOTFS}" \ + --rootfs "${rootfs}" \ --cpu-millicores 100 \ --memory-mb 256 \ /bin/sh -c 'sleep 2; exit 23')" @@ -1882,11 +2132,11 @@ run_firecracker_checks() { sbox_cmd delete "${SANDBOX_ID}" SANDBOX_ID="" - run_dnat_check firecracker "Firecracker" "${EROFS_ROOTFS}" 256 + run_dnat_check firecracker "Firecracker" "${rootfs}" 256 - run_checkpoint_restore_check firecracker "${EROFS_ROOTFS}" - run_storage_quota_check firecracker "${EROFS_ROOTFS}" - run_stress_checks firecracker "${EROFS_ROOTFS}" + run_checkpoint_restore_check firecracker "${rootfs}" + run_storage_quota_check firecracker "${rootfs}" + run_stress_checks firecracker "${rootfs}" } run_runsc_checks() { @@ -2063,6 +2313,11 @@ run_e2e() { prepare_rootfs start_sandboxd start_gateway_httpd + if [ "${STRESS_ONLY}" = "1" ]; then + run_stress_checks firecracker "${STRESS_ROOTFS:-${ROOTFS}}" + log "e2e stress passed" + return + fi if [ "${DISABLE_CGROUP}" = "1" ]; then run_cgroup_disabled_checks else diff --git a/test/e2e/firecracker-virtiofs.Dockerfile b/test/e2e/firecracker-virtiofs.Dockerfile new file mode 100644 index 0000000..c82d411 --- /dev/null +++ b/test/e2e/firecracker-virtiofs.Dockerfile @@ -0,0 +1,65 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + busybox-static \ + ca-certificates \ + e2fsprogs \ + erofs-utils \ + iproute2 \ + ipset \ + iptables \ + iputils-ping \ + jq \ + kmod \ + mount \ + netcat-openbsd \ + procps \ + xfsprogs && \ + rm -rf /var/lib/apt/lists/* && \ + if [ -x /usr/sbin/iptables-legacy ]; then \ + update-alternatives --set iptables /usr/sbin/iptables-legacy; \ + update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy; \ + fi + +COPY output/sandboxd /usr/local/bin/sandboxd +COPY output/sbox /usr/local/bin/sbox +COPY output/oom-hog /usr/local/bin/oom-hog +COPY output/network-policy-client /usr/local/bin/network-policy-client +COPY output/checkpoint-restore /usr/local/bin/checkpoint-restore +COPY output/firecracker /usr/local/bin/firecracker +COPY output/virtiofsd /usr/local/bin/virtiofsd +COPY output/firecracker-vmlinux /opt/firecracker/vmlinux +COPY output/firecracker-initrd.img /opt/firecracker/initrd.img +COPY test/e2e/e2e-run.sh /usr/local/bin/sandboxd-e2e-run + +RUN chmod 0755 \ + /usr/local/bin/sandboxd \ + /usr/local/bin/sbox \ + /usr/local/bin/oom-hog \ + /usr/local/bin/network-policy-client \ + /usr/local/bin/checkpoint-restore \ + /usr/local/bin/firecracker \ + /usr/local/bin/virtiofsd \ + /usr/local/bin/sandboxd-e2e-run && \ + chmod 0644 \ + /opt/firecracker/vmlinux \ + /opt/firecracker/initrd.img + +ENTRYPOINT ["/usr/local/bin/sandboxd-e2e-run"] diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 3a607f7..aebcec4 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -28,9 +28,13 @@ RUNC_BINARY="${RUNC_BINARY:-}" FIRECRACKER_BINARY="${FIRECRACKER_BINARY:-}" FIRECRACKER_KERNEL="${FIRECRACKER_KERNEL:-}" FIRECRACKER_INITRD="${FIRECRACKER_INITRD:-}" +FIRECRACKER_VIRTIOFSD="${FIRECRACKER_VIRTIOFSD:-}" KATA_ROOT="${KATA_ROOT:-}" E2E_STRESS_ROUNDS="${E2E_STRESS_ROUNDS:-0}" E2E_STRESS_CONCURRENCY="${E2E_STRESS_CONCURRENCY:-8}" +E2E_STRESS_ROOTFS_HOST="${E2E_STRESS_ROOTFS_HOST:-}" +E2E_STRESS_CHECKPOINT="${E2E_STRESS_CHECKPOINT:-0}" +E2E_STRESS_ONLY="${E2E_STRESS_ONLY:-0}" E2E_CPU_LIMIT_MODE="${E2E_CPU_LIMIT_MODE:-quota}" E2E_RUNTIME="${E2E_RUNTIME:-all}" E2E_RUNC_ONLY="${E2E_RUNC_ONLY:-0}" @@ -38,6 +42,9 @@ E2E_RUNSC_PLATFORM="${E2E_RUNSC_PLATFORM:-systrap}" E2E_RUN_CGROUP_DISABLED="${E2E_RUN_CGROUP_DISABLED:-1}" E2E_SKIP_BUILD="${E2E_SKIP_BUILD:-0}" E2E_NETWORK_SOAK="${E2E_NETWORK_SOAK:-0}" +E2E_FIRECRACKER_VIRTIOFS="${E2E_FIRECRACKER_VIRTIOFS:-0}" +E2E_HOME_FIXTURE_PARENT="${E2E_HOME_FIXTURE_PARENT:-/var/tmp}" +E2E_KEEP_HOME_FIXTURE="${E2E_KEEP_HOME_FIXTURE:-0}" REDIS_IMAGE="${E2E_REDIS_IMAGE:-docker.io/library/redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf}" REDIS_DNAT_HOST_PORT="${E2E_REDIS_DNAT_HOST_PORT:-18379}" REDIS_CONTAINER="${CONTAINER}-redis" @@ -94,11 +101,19 @@ cleanup_container() { REDIS_FIXTURE_DIR="" fi if [ -n "${SANDBOXD_HOME_FIXTURE_DIR}" ]; then - cleanup_disk_fixture "${SANDBOXD_HOME_FIXTURE_DIR}" + if [ "${E2E_KEEP_HOME_FIXTURE}" = "1" ]; then + log "retained sandboxd home fixture: ${SANDBOXD_HOME_FIXTURE_DIR}" + else + cleanup_disk_fixture "${SANDBOXD_HOME_FIXTURE_DIR}" + fi SANDBOXD_HOME_FIXTURE_DIR="" fi if [ -n "${DISABLED_HOME_FIXTURE_DIR}" ]; then - cleanup_disk_fixture "${DISABLED_HOME_FIXTURE_DIR}" + if [ "${E2E_KEEP_HOME_FIXTURE}" = "1" ]; then + log "retained disabled-cgroup home fixture: ${DISABLED_HOME_FIXTURE_DIR}" + else + cleanup_disk_fixture "${DISABLED_HOME_FIXTURE_DIR}" + fi DISABLED_HOME_FIXTURE_DIR="" fi if [ "${E2E_SKIP_BUILD}" = "0" ] && [ "${E2E_RUNTIME}" = "kata" ]; then @@ -110,7 +125,9 @@ prepare_sandboxd_home_fixture() { local dir local fs_type - dir="$(mktemp -d /var/tmp/sandboxd-e2e-home.XXXXXX)" + [ -d "${E2E_HOME_FIXTURE_PARENT}" ] || + fail "E2E_HOME_FIXTURE_PARENT is not a directory: ${E2E_HOME_FIXTURE_PARENT}" + dir="$(mktemp -d "${E2E_HOME_FIXTURE_PARENT%/}/sandboxd-e2e-home.XXXXXX")" fs_type="$(stat -f -c %T "${dir}")" if [ "${fs_type}" = "tmpfs" ]; then rmdir "${dir}" @@ -231,6 +248,46 @@ case "${E2E_NETWORK_SOAK}" in 0|1) ;; *) fail "E2E_NETWORK_SOAK must be 0 or 1" ;; esac +case "${E2E_FIRECRACKER_VIRTIOFS}" in + 0|1) ;; + *) fail "E2E_FIRECRACKER_VIRTIOFS must be 0 or 1" ;; +esac +case "${E2E_STRESS_CHECKPOINT}" in + 0|1) ;; + *) fail "E2E_STRESS_CHECKPOINT must be 0 or 1" ;; +esac +case "${E2E_STRESS_ONLY}" in + 0|1) ;; + *) fail "E2E_STRESS_ONLY must be 0 or 1" ;; +esac +case "${E2E_KEEP_HOME_FIXTURE}" in + 0|1) ;; + *) fail "E2E_KEEP_HOME_FIXTURE must be 0 or 1" ;; +esac +if [ "${E2E_FIRECRACKER_VIRTIOFS}" = "1" ] && + [ "${E2E_RUNTIME}" != "firecracker" ]; then + fail "E2E_FIRECRACKER_VIRTIOFS requires E2E_RUNTIME=firecracker" +fi +if [ -n "${E2E_STRESS_ROOTFS_HOST}" ]; then + [ -d "${E2E_STRESS_ROOTFS_HOST}" ] || + fail "E2E_STRESS_ROOTFS_HOST is not a directory" + [ "${E2E_RUNTIME}" = "firecracker" ] && + [ "${E2E_FIRECRACKER_VIRTIOFS}" = "1" ] || + fail "E2E_STRESS_ROOTFS_HOST requires Firecracker virtio-fs" +fi +if [ "${E2E_STRESS_CHECKPOINT}" = "1" ] && { + [ "${E2E_RUNTIME}" != "firecracker" ] || + [ "${E2E_FIRECRACKER_VIRTIOFS}" != "1" ]; +}; then + fail "E2E_STRESS_CHECKPOINT requires Firecracker virtio-fs" +fi +if [ "${E2E_STRESS_ONLY}" = "1" ] && { + [ "${E2E_RUNTIME}" != "firecracker" ] || + [ "${E2E_FIRECRACKER_VIRTIOFS}" != "1" ] || + [ "${E2E_STRESS_ROUNDS}" = "0" ]; +}; then + fail "E2E_STRESS_ONLY requires Firecracker virtio-fs stress rounds" +fi if [ "${E2E_NETWORK_SOAK}" = "1" ] && [ "${E2E_RUNTIME}" = "all" ]; then fail "E2E_NETWORK_SOAK requires one selected runtime" fi @@ -300,6 +357,18 @@ if [ "${E2E_SKIP_BUILD}" = "0" ]; then fail "FIRECRACKER_KERNEL is not a file" [ -f "${FIRECRACKER_INITRD}" ] || fail "FIRECRACKER_INITRD is not a file" + if [ "${E2E_FIRECRACKER_VIRTIOFS}" = "1" ]; then + if [ -z "${FIRECRACKER_VIRTIOFSD}" ]; then + for candidate in output/virtiofsd /usr/local/bin/virtiofsd; do + if [ -x "${candidate}" ]; then + FIRECRACKER_VIRTIOFSD="${candidate}" + break + fi + done + fi + [ -x "${FIRECRACKER_VIRTIOFSD}" ] || + fail "FIRECRACKER_VIRTIOFSD is not executable" + fi [ -c /dev/kvm ] || fail "Firecracker e2e requires /dev/kvm" fi fi @@ -355,6 +424,14 @@ if [ "${E2E_SKIP_BUILD}" = "0" ]; then install -m 0755 "${FIRECRACKER_BINARY}" output/firecracker install -m 0644 "${FIRECRACKER_KERNEL}" output/firecracker-vmlinux install -m 0644 "${FIRECRACKER_INITRD}" output/firecracker-initrd.img + if [ "${E2E_FIRECRACKER_VIRTIOFS}" = "1" ]; then + DOCKERFILE="test/e2e/firecracker-virtiofs.Dockerfile" + CGO_ENABLED=0 GOWORK=off GOCACHE=/tmp/go-build \ + GOMODCACHE=/tmp/go-mod-official GOTOOLCHAIN=auto \ + go build -o output/checkpoint-restore \ + ./test/e2e/checkpoint-restore + install -m 0755 "${FIRECRACKER_VIRTIOFSD}" output/virtiofsd + fi fi log "building e2e image ${IMAGE}" @@ -370,6 +447,13 @@ SANDBOXD_HOME_FIXTURE_DIR="$(prepare_sandboxd_home_fixture)" container_network_args=(--net bridge) network_soak_args=(-e E2E_NETWORK_SOAK=0) +stress_rootfs_args=() +if [ -n "${E2E_STRESS_ROOTFS_HOST}" ]; then + stress_rootfs_args=( + -e E2E_STRESS_ROOTFS=/e2e-stress-rootfs + -v "${E2E_STRESS_ROOTFS_HOST}:/e2e-stress-rootfs:ro" + ) +fi if [ "${E2E_NETWORK_SOAK}" = "1" ]; then container_network_args=(--network "${REDIS_NETWORK}") network_soak_args=( @@ -391,12 +475,16 @@ set +e "${container_network_args[@]}" \ -e "E2E_STRESS_ROUNDS=${E2E_STRESS_ROUNDS}" \ -e "E2E_STRESS_CONCURRENCY=${E2E_STRESS_CONCURRENCY}" \ + -e "E2E_STRESS_CHECKPOINT=${E2E_STRESS_CHECKPOINT}" \ + -e "E2E_STRESS_ONLY=${E2E_STRESS_ONLY}" \ -e "E2E_CPU_LIMIT_MODE=${E2E_CPU_LIMIT_MODE}" \ -e "E2E_RUNTIME=${E2E_RUNTIME}" \ -e "E2E_RUNSC_PLATFORM=${E2E_RUNSC_PLATFORM}" \ -e "E2E_FIRECRACKER_CHECKPOINT_MODE=${E2E_FIRECRACKER_CHECKPOINT_MODE:-}" \ + -e "E2E_FIRECRACKER_VIRTIOFS=${E2E_FIRECRACKER_VIRTIOFS}" \ -e "E2E_OCI_ROOTFS_IMAGE=${REDIS_IMAGE}" \ "${network_soak_args[@]}" \ + "${stress_rootfs_args[@]}" \ -v "${SANDBOXD_HOME_FIXTURE_DIR}:/home/akernel:rw" \ --tmpfs /e2e:rw,exec,size=512m \ -v /sys/fs/cgroup:/sys/fs/cgroup:rw \ diff --git a/third_party/runtime-versions.env b/third_party/runtime-versions.env index b7853ea..c2df529 100644 --- a/third_party/runtime-versions.env +++ b/third_party/runtime-versions.env @@ -28,9 +28,9 @@ KATA_AMD64_SHA256=2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a KATA_RELEASE_BASE_URL=https://github.com/kata-containers/kata-containers/releases/download # Promoted akernel-dev/firecracker release: the candidate built from -# develop @e97a6fc23 (vmm.source=fork-source-build) and tested with the +# develop @19dccdd95 (vmm.source=fork-source-build) and tested with the # sandboxd runtime suite before promotion; the published bytes match the # candidate artifact exactly. -FIRECRACKER_RELEASE=v1.16.1-akernel.1 -FIRECRACKER_AMD64_SHA256=f4011842ac37de519e1fb04617b1c57fd8e84429a3e2086133ad6a6446ac8c94 -FIRECRACKER_AMD64_URL=https://github.com/akernel-dev/firecracker/releases/download/v1.16.1-akernel.1/firecracker-v1.16.1-akernel.1-x86_64.tgz +FIRECRACKER_RELEASE=v1.16.1-akernel.2 +FIRECRACKER_AMD64_SHA256=f565655926c59bbb72b054b8d5f295169c91322e449ea57591aa5784fa082969 +FIRECRACKER_AMD64_URL=https://github.com/akernel-dev/firecracker/releases/download/v1.16.1-akernel.2/firecracker-v1.16.1-akernel.2-x86_64.tgz