diff --git a/cmd/firecracker-agent/main.go b/cmd/firecracker-agent/main.go index ab25773..c9e57cf 100644 --- a/cmd/firecracker-agent/main.go +++ b/cmd/firecracker-agent/main.go @@ -44,6 +44,7 @@ const ( containerMountRoot = "/container/root" containerLower = "/container/lower" containerOverlay = "/container/overlay" + containerNative = "/container/overlay/native" sandboxInitMode = "sandbox-init" sandboxConfigFD = 3 sandboxStatusFD = 4 @@ -359,6 +360,15 @@ func configure(request firecrackerproto.ConfigureRequest) error { return fmt.Errorf("create mount target %s: %w", mount.Target, err) } } + for _, mount := range request.NativeWritableMounts { + if _, err := ensureContainerDirectory(mount.Target); err != nil { + return fmt.Errorf( + "create native writable mount target %s: %w", + mount.Target, + err, + ) + } + } for _, file := range request.Files { if err := injectFile(file); err != nil { return err @@ -372,6 +382,11 @@ func configure(request firecrackerproto.ConfigureRequest) error { return err } } + for index, mount := range request.NativeWritableMounts { + if err := mountNativeWritable(index, mount); err != nil { + return err + } + } if request.RootReadonly { if err := unix.Mount( "", @@ -891,6 +906,48 @@ func mountGuestFilesystem(mount firecrackerproto.MountSpec) error { } } +func mountNativeWritable( + index int, + mount firecrackerproto.NativeWritableMountSpec, +) error { + return mountNativeWritableUnder( + containerNative, + containerRoot, + index, + mount, + unix.Mount, + ) +} + +func mountNativeWritableUnder( + sourceRoot, + root string, + index int, + mount firecrackerproto.NativeWritableMountSpec, + mountFn func(string, string, string, uintptr, string) error, +) error { + if index < 0 { + return fmt.Errorf("native writable mount index %d is negative", index) + } + source := filepath.Join(sourceRoot, strconv.Itoa(index)) + if err := os.MkdirAll(source, 0755); err != nil { + return fmt.Errorf("create native writable mount source %s: %w", source, err) + } + target, err := ensureContainerDirectoryUnder(root, mount.Target) + if err != nil { + return err + } + if err := mountFn(source, target, "", unix.MS_BIND, ""); err != nil { + return fmt.Errorf( + "bind native writable mount %s at %s: %w", + source, + mount.Target, + err, + ) + } + return nil +} + func mountGuestEROFS(mount firecrackerproto.MountSpec) error { if mount.Device == "" { return errors.New("EROFS guest mount device is empty") diff --git a/cmd/firecracker-agent/main_test.go b/cmd/firecracker-agent/main_test.go index 9989ee7..138480f 100644 --- a/cmd/firecracker-agent/main_test.go +++ b/cmd/firecracker-agent/main_test.go @@ -231,6 +231,68 @@ func TestEnsureContainerDirectoryRejectsSymlink(t *testing.T) { } } +func TestMountNativeWritableUnder(t *testing.T) { + sourceRoot := filepath.Join(t.TempDir(), "native") + root := t.TempDir() + var source, target string + var flags uintptr + err := mountNativeWritableUnder( + sourceRoot, + root, + 2, + firecrackerproto.NativeWritableMountSpec{Target: "/var/lib/docker"}, + func(gotSource, gotTarget, fsType string, gotFlags uintptr, data string) error { + source = gotSource + target = gotTarget + flags = gotFlags + if fsType != "" || data != "" { + t.Fatalf("bind mount arguments = %q, %q", fsType, data) + } + return nil + }, + ) + if err != nil { + t.Fatal(err) + } + if source != filepath.Join(sourceRoot, "2") || + target != filepath.Join(root, "var/lib/docker") || + flags != unix.MS_BIND { + t.Fatalf("bind mount = %q, %q, %#x", source, target, flags) + } + for _, path := range []string{source, target} { + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + t.Fatalf("native writable directory %q = %+v, %v", path, info, err) + } + } +} + +func TestMountNativeWritableUnderRejectsSymlinkTarget(t *testing.T) { + sourceRoot := filepath.Join(t.TempDir(), "native") + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "var")); err != nil { + t.Fatal(err) + } + mounted := false + err := mountNativeWritableUnder( + sourceRoot, + root, + 0, + firecrackerproto.NativeWritableMountSpec{Target: "/var/lib/docker"}, + func(string, string, string, uintptr, string) error { + mounted = true + return nil + }, + ) + if err == nil || !strings.Contains(err.Error(), "traverses symlink") { + t.Fatalf("symlink error = %v", err) + } + if mounted { + t.Fatal("native writable mount was attempted for a symlink target") + } +} + func TestPrepareContainerFileReplacesFinalSymlink(t *testing.T) { root := t.TempDir() if err := os.Mkdir(filepath.Join(root, "etc"), 0755); err != nil { diff --git a/cmd/sbox/start.go b/cmd/sbox/start.go index 9620a80..58a38b6 100644 --- a/cmd/sbox/start.go +++ b/cmd/sbox/start.go @@ -55,6 +55,10 @@ var StartCmd = cli.Command{ Name: "enable-kvm", Usage: "expose the configured KVM device in a runc sandbox", }, + cli.StringFlag{ + Name: "extra-config", + Usage: "runtime-specific configuration as a JSON object", + }, cli.StringFlag{ Name: "cwd", Usage: "working directory inside the sandbox", @@ -139,7 +143,10 @@ var StartCmd = cli.Command{ if err != nil { return err } - extraConfig, err := startExtraConfig(context.Bool("enable-kvm")) + extraConfig, err := startExtraConfig( + context.String("extra-config"), + context.Bool("enable-kvm"), + ) if err != nil { return err } @@ -209,13 +216,23 @@ func startRootfs(localPath, imageURL string, readonly bool) (*runtime.RootfsConf return rootfs, nil } -func startExtraConfig(enableKVM bool) (string, error) { - if !enableKVM { +func startExtraConfig(raw string, enableKVM bool) (string, error) { + if strings.TrimSpace(raw) == "" && !enableKVM { return "", nil } - data, err := json.Marshal(struct { - EnableKVM bool `json:"enableKVM"` - }{EnableKVM: true}) + config := make(map[string]json.RawMessage) + if strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), &config); err != nil { + return "", fmt.Errorf("decode --extra-config: %w", err) + } + if config == nil { + return "", fmt.Errorf("--extra-config must be a JSON object") + } + } + if enableKVM { + config["enableKVM"] = json.RawMessage("true") + } + data, err := json.Marshal(config) if err != nil { return "", err } diff --git a/cmd/sbox/start_test.go b/cmd/sbox/start_test.go index ff7babe..86e6d5a 100644 --- a/cmd/sbox/start_test.go +++ b/cmd/sbox/start_test.go @@ -62,13 +62,38 @@ func TestStorageMBToBytes(t *testing.T) { } func TestStartExtraConfig(t *testing.T) { - value, err := startExtraConfig(false) + value, err := startExtraConfig("", false) require.NoError(t, err) assert.Empty(t, value) - value, err = startExtraConfig(true) + value, err = startExtraConfig("", true) require.NoError(t, err) assert.JSONEq(t, `{"enableKVM":true}`, value) + + value, err = startExtraConfig( + `{"nativeWritableMounts":[{"target":"/var/lib/docker"}]}`, + false, + ) + require.NoError(t, err) + assert.JSONEq(t, + `{"nativeWritableMounts":[{"target":"/var/lib/docker"}]}`, + value, + ) + + value, err = startExtraConfig( + `{"enableKVM":false,"runtimeOption":"kept"}`, + true, + ) + require.NoError(t, err) + assert.JSONEq(t, + `{"enableKVM":true,"runtimeOption":"kept"}`, + value, + ) + + for _, invalid := range []string{"null", "[]", "not-json"} { + _, err = startExtraConfig(invalid, false) + require.Error(t, err) + } } func TestStartRootfs(t *testing.T) { diff --git a/doc/checkpoint-restore.md b/doc/checkpoint-restore.md index 7809fe3..1856b7a 100644 --- a/doc/checkpoint-restore.md +++ b/doc/checkpoint-restore.md @@ -214,6 +214,13 @@ image. The artifact overlay must not be used as the restored VM's writable image: checkpoint generations are immutable, the source may keep running, and concurrent restores require independent writable layers. +Firecracker native writable mounts do not add checkpoint components. Their +directories reside in the same `overlay.ext4` as the root overlay's upper and +work directories, while the VM snapshot preserves the guest bind mounts. A +checkpoint and restore therefore carries their data and mount state through +the existing `overlay.ext4`, `memory`, and `vmstate` artifacts and applies the +same quota, clone, durability, and storage-placement behavior. + sandboxd uses `FICLONE` for these copies when possible: - the live writable image under diff --git a/doc/runtime.md b/doc/runtime.md index afc2c34..cf0a5ac 100644 --- a/doc/runtime.md +++ b/doc/runtime.md @@ -102,6 +102,28 @@ and mount setup. An explicit writable-layer limit sizes this image, with a 16 MiB minimum. Without an explicit limit, `default_overlay_size_bytes` applies and defaults to 10 GiB. The image is removed on sandbox deletion. +A Firecracker start may expose directories from that same private ext4 image +at selected guest paths. This is useful for workloads such as Docker that need +a native filesystem instead of placing their own overlay on sandboxd's root +OverlayFS. Configure the paths through the runtime-specific start configuration: + +```json +{ + "nativeWritableMounts": [ + {"target": "/var/lib/docker"} + ] +} +``` + +Each target is backed by a distinct, root-owned directory next to the root +overlay's `upper` and `work` directories and is bind-mounted into the guest. +It is not a host bind mount or an additional Firecracker drive. The root +overlay and every native writable mount therefore share the single ext4 image +and its writable-layer quota. Targets must be canonical absolute directory +paths, may not overlap each other, an ordinary mount, or the guest's `/dev`, +`/proc`, `/run`, `/sys`, and `/tmp` system mounts. At most 16 targets may be +requested. + EROFS and `rofs` mounts must also name regular EROFS image files and are attached as read-only drives. Read-only regular files are injected into the guest, limited to 1 MiB per file and 4 MiB in total; this narrow path supports @@ -113,6 +135,6 @@ instead of being silently weakened. Private tmpfs mounts are supported with a bounded set of standard security, ownership, mode, inode, and size options. -The private ext4 image remains in the filestore across sandboxd restart so the -handler can recover the running VMM. It is cleaned by normal or idempotent -sandbox deletion. +The private ext4 image, including native writable mount data, remains in the +filestore across sandboxd restart so the handler can recover the running VMM. +It is cleaned by normal or idempotent sandbox deletion. diff --git a/internal/firecrackerproto/protocol.go b/internal/firecrackerproto/protocol.go index cc10ecf..84619ed 100644 --- a/internal/firecrackerproto/protocol.go +++ b/internal/firecrackerproto/protocol.go @@ -104,6 +104,12 @@ type MountSpec struct { Options []string `json:"options,omitempty"` } +// NativeWritableMountSpec exposes a directory from the sandbox's private +// ext4 writable layer at a distinct path inside the guest root filesystem. +type NativeWritableMountSpec struct { + Target string `json:"target"` +} + type FileSpec struct { Target string `json:"target"` Content []byte `json:"content"` @@ -112,14 +118,15 @@ type FileSpec struct { } type ConfigureRequest struct { - Hostname string `json:"hostname"` - RootDevice string `json:"root_device"` - OverlayDevice string `json:"overlay_device"` - RootReadonly bool `json:"root_readonly,omitempty"` - Process ProcessSpec `json:"process"` - Network NetworkSpec `json:"network"` - Mounts []MountSpec `json:"mounts,omitempty"` - Files []FileSpec `json:"files,omitempty"` + Hostname string `json:"hostname"` + RootDevice string `json:"root_device"` + OverlayDevice string `json:"overlay_device"` + RootReadonly bool `json:"root_readonly,omitempty"` + Process ProcessSpec `json:"process"` + Network NetworkSpec `json:"network"` + Mounts []MountSpec `json:"mounts,omitempty"` + NativeWritableMounts []NativeWritableMountSpec `json:"native_writable_mounts,omitempty"` + Files []FileSpec `json:"files,omitempty"` } type ExecRequest struct { diff --git a/internal/firecrackerproto/protocol_test.go b/internal/firecrackerproto/protocol_test.go index fb7e38a..ba5abed 100644 --- a/internal/firecrackerproto/protocol_test.go +++ b/internal/firecrackerproto/protocol_test.go @@ -64,6 +64,33 @@ func TestMessageRoundTripWithShortWrites(t *testing.T) { } } +func TestConfigureRequestRoundTripIncludesNativeWritableMounts(t *testing.T) { + var encoded bytes.Buffer + request := ConfigureRequest{ + NativeWritableMounts: []NativeWritableMountSpec{{ + Target: "/var/lib/docker", + }}, + } + if err := WriteMessage(&encoded, MessageConfigure, request); err != nil { + t.Fatal(err) + } + messageType, payload, err := ReadMessage(&encoded) + if err != nil { + t.Fatal(err) + } + if messageType != MessageConfigure { + t.Fatalf("message type = %d, want %d", messageType, MessageConfigure) + } + var decoded ConfigureRequest + if err := Decode(payload, &decoded); err != nil { + t.Fatal(err) + } + if len(decoded.NativeWritableMounts) != 1 || + decoded.NativeWritableMounts[0].Target != "/var/lib/docker" { + t.Fatalf("decoded request = %+v", decoded) + } +} + func TestDecodeRejectsUnknownFields(t *testing.T) { var request ExecRequest if err := Decode([]byte(`{"command":"true","unexpected":1}`), &request); err == nil { diff --git a/internal/server/server.go b/internal/server/server.go index 187248a..430990b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1545,6 +1545,7 @@ func (h *sandboxService) Start(ctx context.Context, request *runtime.StartReques DisableCgroup: h.config.DisableCgroup, SpecUpdates: specUpdates, WritableLayerLimitBytes: startReq.WritableLayerLimitBytes, + ExtraConfig: startReq.ExtraConfig, EnableKVM: extraConfig.EnableKVM, CheckpointDir: checkpointDir, } diff --git a/pkg/runtime/firecracker/extra_config.go b/pkg/runtime/firecracker/extra_config.go new file mode 100644 index 0000000..bbdf45d --- /dev/null +++ b/pkg/runtime/firecracker/extra_config.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firecracker + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" +) + +const firecrackerMaxNativeWritableMounts = 16 + +type firecrackerNativeWritableMount struct { + Target string `json:"target"` +} + +type firecrackerExtraConfig struct { + NativeWritableMounts []firecrackerNativeWritableMount `json:"nativeWritableMounts,omitempty"` +} + +func parseFirecrackerExtraConfig(value string) (firecrackerExtraConfig, error) { + config := firecrackerExtraConfig{} + if strings.TrimSpace(value) == "" { + return config, nil + } + if err := json.Unmarshal([]byte(value), &config); err != nil { + return config, fmt.Errorf("decode Firecracker extra config: %w", err) + } + return config, nil +} + +func validateFirecrackerNativeWritableMounts( + mounts []firecrackerNativeWritableMount, + existingTargets []string, +) error { + if len(mounts) > firecrackerMaxNativeWritableMounts { + return fmt.Errorf( + "Firecracker supports at most %d native writable mounts", + firecrackerMaxNativeWritableMounts, + ) + } + + for index, mount := range mounts { + target := mount.Target + clean := filepath.Clean(target) + if !filepath.IsAbs(target) || clean == "/" || clean != target { + return fmt.Errorf( + "invalid Firecracker native writable mount target %q", + target, + ) + } + for _, reserved := range []string{"/dev", "/proc", "/run", "/sys", "/tmp"} { + if firecrackerMountTargetsOverlap(clean, reserved) { + return fmt.Errorf( + "Firecracker native writable mount target %q conflicts with system mount %s", + target, + reserved, + ) + } + } + for previous := 0; previous < index; previous++ { + if firecrackerMountTargetsOverlap(clean, mounts[previous].Target) { + return fmt.Errorf( + "Firecracker native writable mount targets %q and %q overlap", + mounts[previous].Target, + target, + ) + } + } + for _, existing := range existingTargets { + if existing == "" { + continue + } + if firecrackerMountTargetsOverlap(clean, filepath.Clean(existing)) { + return fmt.Errorf( + "Firecracker native writable mount target %q overlaps mount target %q", + target, + existing, + ) + } + } + } + return nil +} + +func firecrackerMountTargetsOverlap(left, right string) bool { + left = strings.TrimSuffix(filepath.Clean(left), "/") + "/" + right = strings.TrimSuffix(filepath.Clean(right), "/") + "/" + return strings.HasPrefix(left, right) || strings.HasPrefix(right, left) +} diff --git a/pkg/runtime/firecracker/extra_config_test.go b/pkg/runtime/firecracker/extra_config_test.go new file mode 100644 index 0000000..48e82c4 --- /dev/null +++ b/pkg/runtime/firecracker/extra_config_test.go @@ -0,0 +1,108 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firecracker + +import ( + "fmt" + "strings" + "testing" +) + +func TestParseFirecrackerExtraConfig(t *testing.T) { + config, err := parseFirecrackerExtraConfig( + `{"networkStack":"sandbox","nativeWritableMounts":[` + + `{"target":"/var/lib/docker"}]}`, + ) + if err != nil { + t.Fatal(err) + } + if len(config.NativeWritableMounts) != 1 || + config.NativeWritableMounts[0].Target != "/var/lib/docker" { + t.Fatalf("native writable mounts = %+v", config.NativeWritableMounts) + } +} + +func TestValidateFirecrackerNativeWritableMounts(t *testing.T) { + valid := []firecrackerNativeWritableMount{ + {Target: "/var/lib/docker"}, + {Target: "/home/cache"}, + } + if err := validateFirecrackerNativeWritableMounts(valid, []string{"/mnt/data"}); err != nil { + t.Fatalf("valid native writable mounts rejected: %v", err) + } +} + +func TestValidateFirecrackerNativeWritableMountsRejectsInvalidTargets(t *testing.T) { + tests := []struct { + name string + mounts []firecrackerNativeWritableMount + existing []string + message string + }{ + { + name: "relative", + mounts: []firecrackerNativeWritableMount{{Target: "var/lib/docker"}}, + message: "invalid", + }, + { + name: "root", + mounts: []firecrackerNativeWritableMount{{Target: "/"}}, + message: "invalid", + }, + { + name: "not canonical", + mounts: []firecrackerNativeWritableMount{{Target: "/var/../data"}}, + message: "invalid", + }, + { + name: "system mount", + mounts: []firecrackerNativeWritableMount{{Target: "/run/docker"}}, + message: "system mount", + }, + { + name: "native overlap", + mounts: []firecrackerNativeWritableMount{ + {Target: "/var/lib"}, + {Target: "/var/lib/docker"}, + }, + message: "overlap", + }, + { + name: "regular mount overlap", + mounts: []firecrackerNativeWritableMount{{Target: "/mnt/data/cache"}}, + existing: []string{"/mnt/data"}, + message: "overlaps mount target", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateFirecrackerNativeWritableMounts(test.mounts, test.existing) + if err == nil || !strings.Contains(err.Error(), test.message) { + t.Fatalf("validation error = %v, want %q", err, test.message) + } + }) + } +} + +func TestValidateFirecrackerNativeWritableMountsRejectsTooMany(t *testing.T) { + mounts := make([]firecrackerNativeWritableMount, firecrackerMaxNativeWritableMounts+1) + for index := range mounts { + mounts[index].Target = fmt.Sprintf("/data/%d", index) + } + err := validateFirecrackerNativeWritableMounts(mounts, nil) + if err == nil || !strings.Contains(err.Error(), "at most") { + t.Fatalf("validation error = %v", err) + } +} diff --git a/pkg/runtime/firecracker/handler.go b/pkg/runtime/firecracker/handler.go index 4612645..8adeb03 100644 --- a/pkg/runtime/firecracker/handler.go +++ b/pkg/runtime/firecracker/handler.go @@ -266,6 +266,22 @@ func (handler *Handler) CheckpointRestoreCapabilities() runtimecore.CheckpointRe func (handler *Handler) ValidateStartRequest( request *runtimeapi.StartRequest, ) error { + extraConfig, err := parseFirecrackerExtraConfig(request.GetExtraConfig()) + if err != nil { + return err + } + mountTargets := make([]string, 0, len(request.GetMounts())) + for _, mount := range request.GetMounts() { + if mount != nil { + mountTargets = append(mountTargets, mount.GetTarget()) + } + } + if err := validateFirecrackerNativeWritableMounts( + extraConfig.NativeWritableMounts, + mountTargets, + ); err != nil { + return err + } if rootfs := request.GetRootfs(); rootfs != nil && (rootfs.GetType() == runtimeapi.RootfsSrcType_IMAGE || rootfs.GetImageUrl() != "") { diff --git a/pkg/runtime/firecracker/handler_test.go b/pkg/runtime/firecracker/handler_test.go index 3089abf..fb1caee 100644 --- a/pkg/runtime/firecracker/handler_test.go +++ b/pkg/runtime/firecracker/handler_test.go @@ -157,6 +157,30 @@ func TestFirecrackerValidateStartRequestAllowsEnabledOCIRootfs(t *testing.T) { } } +func TestFirecrackerValidateStartRequestAcceptsNativeWritableMount(t *testing.T) { + handler := &Handler{} + request := &runtimeapi.StartRequest{ + ExtraConfig: `{"nativeWritableMounts":[` + + `{"target":"/var/lib/docker"}]}`, + } + if err := handler.ValidateStartRequest(request); err != nil { + t.Fatalf("ValidateStartRequest() error = %v", err) + } +} + +func TestFirecrackerValidateStartRequestRejectsNativeWritableMountOverlap(t *testing.T) { + handler := &Handler{} + request := &runtimeapi.StartRequest{ + ExtraConfig: `{"nativeWritableMounts":[` + + `{"target":"/var/lib/docker"}]}`, + Mounts: []*runtimeapi.Mount{{Target: "/var/lib"}}, + } + err := handler.ValidateStartRequest(request) + if err == nil || !strings.Contains(err.Error(), "overlaps mount target") { + t.Fatalf("ValidateStartRequest() error = %v", err) + } +} + func TestFirecrackerRuntimeDirectoryIsStableAndBounded(t *testing.T) { handler := &Handler{runtimeRoot: "/run/sandboxd/firecracker"} sandboxID := "sbox-" + strings.Repeat("a", 120) diff --git a/pkg/runtime/firecracker/storage.go b/pkg/runtime/firecracker/storage.go index abdcbec..c8d50d7 100644 --- a/pkg/runtime/firecracker/storage.go +++ b/pkg/runtime/firecracker/storage.go @@ -90,6 +90,20 @@ func prepareFirecrackerStorage( if err != nil { return nil, err } + extraConfig, err := parseFirecrackerExtraConfig(startConfig.ExtraConfig) + if err != nil { + return nil, err + } + mountTargets := make([]string, 0, len(mounts)) + for _, mount := range mounts { + mountTargets = append(mountTargets, mount.Destination) + } + if err := validateFirecrackerNativeWritableMounts( + extraConfig.NativeWritableMounts, + mountTargets, + ); err != nil { + return nil, err + } plan := &firecrackerStoragePlan{ rootDrive: firecrackerDrive{ ID: "rootfs", @@ -118,6 +132,12 @@ func prepareFirecrackerStorage( }, }, } + for _, mount := range extraConfig.NativeWritableMounts { + plan.configure.NativeWritableMounts = append( + plan.configure.NativeWritableMounts, + firecrackerproto.NativeWritableMountSpec{Target: mount.Target}, + ) + } injectedBytes := 0 for _, mount := range mounts { diff --git a/pkg/runtime/firecracker/storage_test.go b/pkg/runtime/firecracker/storage_test.go index 2773074..2e6e215 100644 --- a/pkg/runtime/firecracker/storage_test.go +++ b/pkg/runtime/firecracker/storage_test.go @@ -66,6 +66,8 @@ func TestPrepareFirecrackerStorage(t *testing.T) { } plan, err := prepareFirecrackerStorage(spec, runtimecore.StartConfig{ Network: firecrackerTestNetwork(), + ExtraConfig: `{"nativeWritableMounts":[` + + `{"target":"/var/lib/docker"}]}`, }) if err != nil { t.Fatal(err) @@ -87,6 +89,13 @@ func TestPrepareFirecrackerStorage(t *testing.T) { plan.configure.Mounts[1].Target != "/opt/runtime" { t.Fatalf("guest mounts = %+v", plan.configure.Mounts) } + if len(plan.configure.NativeWritableMounts) != 1 || + plan.configure.NativeWritableMounts[0].Target != "/var/lib/docker" { + t.Fatalf( + "native writable mounts = %+v", + plan.configure.NativeWritableMounts, + ) + } if len(plan.configure.Files) != 1 || !plan.configure.Files[0].Readonly || string(plan.configure.Files[0].Content) != "nameserver 1.1.1.1\n" || @@ -105,6 +114,28 @@ func TestPrepareFirecrackerStorage(t *testing.T) { } } +func TestPrepareFirecrackerStorageRejectsNativeWritableMountOverlap(t *testing.T) { + _, err := prepareFirecrackerStorage( + &runtimecore.Spec{ + Root: &runtimecore.Root{Path: fakeEROFSImage(t, "root.erofs")}, + Process: &runtimecore.Process{Args: []string{"/bin/true"}}, + Mounts: []runtimecore.Mount{{ + Type: "tmpfs", + Source: "tmpfs", + Destination: "/var/lib/docker/cache", + }}, + }, + runtimecore.StartConfig{ + Network: firecrackerTestNetwork(), + ExtraConfig: `{"nativeWritableMounts":[` + + `{"target":"/var/lib/docker"}]}`, + }, + ) + if err == nil || !strings.Contains(err.Error(), "overlaps mount target") { + t.Fatalf("overlap error = %v", err) + } +} + func TestPrepareFirecrackerStorageUsesLastMountForTarget(t *testing.T) { first := filepath.Join(t.TempDir(), "first-resolv.conf") if err := os.WriteFile(first, []byte("nameserver 1.1.1.1\n"), 0644); err != nil { diff --git a/pkg/runtime/handler.go b/pkg/runtime/handler.go index 537a089..a51c875 100644 --- a/pkg/runtime/handler.go +++ b/pkg/runtime/handler.go @@ -98,6 +98,7 @@ type StartConfig struct { DisableCgroup bool SpecUpdates *SpecUpdates WritableLayerLimitBytes uint64 + ExtraConfig string EnableKVM bool CheckpointDir string } diff --git a/test/e2e/README.md b/test/e2e/README.md index 7bf5099..0468115 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -29,11 +29,11 @@ The flow: 11. checkpoints the same runsc or Firecracker sandbox ten consecutive times, verifies it keeps running, and restores the tenth artifact; 12. verifies Firecracker's EROFS root and mount contract, OCI-to-EROFS rootfs - conversion, private ext4 overlay, quota exhaustion, guest exec/TTY - protocol, direct service access, local DNAT, network ACL and managed DNS - replacement, crash recovery, stale-policy removal, exit-code recovery when - the daemon is unavailable, and reuse of the same TAP without policy - leakage; and + conversion, private ext4 overlay, native writable ext4 mounts (including + checkpoint/restore), quota exhaustion, guest exec/TTY protocol, direct + service access, local DNAT, network ACL and managed DNS replacement, crash + recovery, stale-policy removal, exit-code recovery when the daemon is + unavailable, and reuse of the same TAP without policy leakage; and 13. runs concurrent Redis SET/GET traffic from every runtime to a sibling Redis container through SNAT and from that container to the sandbox's published port through DNAT. The test verifies the translated source diff --git a/test/e2e/checkpoint-restore/main.go b/test/e2e/checkpoint-restore/main.go index 273f286..e0bf81a 100644 --- a/test/e2e/checkpoint-restore/main.go +++ b/test/e2e/checkpoint-restore/main.go @@ -43,6 +43,7 @@ type options struct { memoryMB float64 cpu int storageMB uint64 + extraConfig string compress bool leaveRunning bool snapshotType string @@ -69,6 +70,8 @@ func main() { flag.Float64Var(&value.memoryMB, "memory-mb", 128, "sandbox memory in MiB") flag.IntVar(&value.cpu, "cpu", 500, "CPU quota (milli-CPU)") flag.Uint64Var(&value.storageMB, "storage-mb", 64, "writable layer in MiB") + flag.StringVar(&value.extraConfig, "extra-config", "", + "runtime-specific configuration as a JSON object") flag.BoolVar(&value.compress, "compress", true, "compress checkpoint artifacts") flag.BoolVar(&value.leaveRunning, "leave-running", true, "leave source running") flag.StringVar(&value.snapshotType, "snapshot-type", "", @@ -155,6 +158,7 @@ func start( "Memory": value.memoryMB, }, WritableLayerLimitBytes: value.storageMB * 1024 * 1024, + ExtraConfig: value.extraConfig, } data, err := protojson.MarshalOptions{ Indent: " ", diff --git a/test/e2e/e2e-run.sh b/test/e2e/e2e-run.sh index 77db30d..905c464 100755 --- a/test/e2e/e2e-run.sh +++ b/test/e2e/e2e-run.sh @@ -735,8 +735,13 @@ run_checkpoint_restore_check() { local checkpoint_dir="" local checkpoint_count=10 local memory_mb=128 + local extra_config_args=() if [ "${runtime}" = "firecracker" ]; then memory_mb=256 + extra_config_args=( + --extra-config + '{"nativeWritableMounts":[{"target":"/var/lib/native-checkpoint"}]}' + ) fi log "testing ${suffix} ${checkpoint_count} consecutive checkpoints and restoring the last" @@ -752,11 +757,21 @@ run_checkpoint_restore_check() { --sandbox-id "${source_id}" \ --request-file "${request_file}" \ --memory-mb "${memory_mb}" \ - --storage-mb 64)" + --storage-mb 64 \ + "${extra_config_args[@]}")" assert_eq "${SANDBOX_ID}" "${source_id}" "${suffix} checkpoint source ID" wait_for_state "${SANDBOX_ID}" "SANDBOX_STATE_RUNNING" 300 sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ 'echo checkpoint-state-ok > /var/checkpoint-persist' + if [ "${runtime}" = "firecracker" ]; then + local native_fstype + native_fstype="$(sbox_cmd exec "${SANDBOX_ID}" /bin/awk \ + '$2 == "/var/lib/native-checkpoint" { print $3 }' /proc/mounts)" + assert_eq "${native_fstype}" "ext4" \ + "${suffix} native writable mount filesystem" + sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ + 'echo native-checkpoint-ok > /var/lib/native-checkpoint/state' + fi local before="" local attempt @@ -860,6 +875,11 @@ run_checkpoint_restore_check() { 'for namespace in mnt pid uts ipc; do test "$(readlink /proc/self/ns/$namespace)" = "$(readlink /proc/1/ns/$namespace)" || exit 1; done; test "$(hostname)" = akernel; cat /proc/1/comm')" assert_eq "${restored_init}" "sh" \ "${suffix} restored exec joined sandbox namespaces" + local restored_native + restored_native="$(sbox_cmd exec "${SANDBOX_ID}" /bin/cat \ + /var/lib/native-checkpoint/state)" + assert_eq "${restored_native}" "native-checkpoint-ok" \ + "${suffix} restored native writable mount" fi local restored_generation restored_generation="$(sbox_cmd exec "${SANDBOX_ID}" \ @@ -903,6 +923,12 @@ run_checkpoint_restore_check() { persisted="$(sbox_cmd exec "${SANDBOX_ID}" /bin/cat /var/checkpoint-persist)" assert_eq "${persisted}" "checkpoint-state-ok" \ "${suffix} target independent of checkpoint directory" + if [ "${runtime}" = "firecracker" ]; then + persisted="$(sbox_cmd exec "${SANDBOX_ID}" /bin/cat \ + /var/lib/native-checkpoint/state)" + assert_eq "${persisted}" "native-checkpoint-ok" \ + "${suffix} native mount independent of checkpoint directory" + fi sbox_cmd delete "${SANDBOX_ID}" SANDBOX_ID="" } @@ -1622,6 +1648,8 @@ run_firecracker_checks() { --mount "${HOST_MOUNT}/input.txt:/mnt/host/input.txt:bind:ro" \ --mount "${EROFS_MOUNT_IMAGE}:/mnt/erofs:erofs:ro" \ --mount "tmpfs:/mnt/ram:tmpfs:rw,nosuid,nodev,noexec,size=1m,mode=0755" \ + --extra-config \ + '{"nativeWritableMounts":[{"target":"/var/lib/docker"}]}' \ --stdout "${main_stdout}" \ --stderr "${main_stderr}" \ --cpu-millicores 1500 \ @@ -1648,6 +1676,12 @@ run_firecracker_checks() { got="$(sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ 'echo firecracker-write-ok > /var/firecracker-write && cat /var/firecracker-write')" assert_eq "${got}" "firecracker-write-ok" "Firecracker writable overlay" + got="$(sbox_cmd exec "${SANDBOX_ID}" /bin/awk \ + '$2 == "/var/lib/docker" { print $3 }' /proc/mounts)" + assert_eq "${got}" "ext4" "Firecracker native writable mount filesystem" + got="$(sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ + 'echo native-write-ok > /var/lib/docker/check && cat /var/lib/docker/check')" + assert_eq "${got}" "native-write-ok" "Firecracker native writable mount" got="$(sbox_cmd exec "${SANDBOX_ID}" /bin/sh -c \ 'echo tmpfs-ok > /mnt/ram/check && cat /mnt/ram/check')" assert_eq "${got}" "tmpfs-ok" "Firecracker private tmpfs mount"