feat(firecracker): add native writable mounts

cmd/firecracker-agent/main.go

@@ -44,6 +44,7 @@ const (
 	containerMountRoot = "/container/root"
 	containerLower     = "/container/lower"
 	containerOverlay   = "/container/overlay"
+	containerNative    = "/container/overlay/native"
 	sandboxInitMode    = "sandbox-init"
 	sandboxConfigFD    = 3
 	sandboxStatusFD    = 4
@@ -359,6 +360,15 @@ func configure(request firecrackerproto.ConfigureRequest) error {
 			return fmt.Errorf("create mount target %s: %w", mount.Target, err)
 		}
 	}
+	for _, mount := range request.NativeWritableMounts {
+		if _, err := ensureContainerDirectory(mount.Target); err != nil {
+			return fmt.Errorf(
+				"create native writable mount target %s: %w",
+				mount.Target,
+				err,
+			)
+		}
+	}
 	for _, file := range request.Files {
 		if err := injectFile(file); err != nil {
 			return err
@@ -372,6 +382,11 @@ func configure(request firecrackerproto.ConfigureRequest) error {
 			return err
 		}
 	}
+	for index, mount := range request.NativeWritableMounts {
+		if err := mountNativeWritable(index, mount); err != nil {
+			return err
+		}
+	}
 	if request.RootReadonly {
 		if err := unix.Mount(
 			"",
@@ -891,6 +906,48 @@ func mountGuestFilesystem(mount firecrackerproto.MountSpec) error {
 	}
 }
 
+func mountNativeWritable(
+	index int,
+	mount firecrackerproto.NativeWritableMountSpec,
+) error {
+	return mountNativeWritableUnder(
+		containerNative,
+		containerRoot,
+		index,
+		mount,
+		unix.Mount,
+	)
+}
+
+func mountNativeWritableUnder(
+	sourceRoot,
+	root string,
+	index int,
+	mount firecrackerproto.NativeWritableMountSpec,
+	mountFn func(string, string, string, uintptr, string) error,
+) error {
+	if index < 0 {
+		return fmt.Errorf("native writable mount index %d is negative", index)
+	}
+	source := filepath.Join(sourceRoot, strconv.Itoa(index))
+	if err := os.MkdirAll(source, 0755); err != nil {
+		return fmt.Errorf("create native writable mount source %s: %w", source, err)
+	}
+	target, err := ensureContainerDirectoryUnder(root, mount.Target)
+	if err != nil {
+		return err
+	}
+	if err := mountFn(source, target, "", unix.MS_BIND, ""); err != nil {
+		return fmt.Errorf(
+			"bind native writable mount %s at %s: %w",
+			source,
+			mount.Target,
+			err,
+		)
+	}
+	return nil
+}
+
 func mountGuestEROFS(mount firecrackerproto.MountSpec) error {
 	if mount.Device == "" {
 		return errors.New("EROFS guest mount device is empty")

cmd/firecracker-agent/main_test.go

@@ -231,6 +231,68 @@ func TestEnsureContainerDirectoryRejectsSymlink(t *testing.T) {
 	}
 }
 
+func TestMountNativeWritableUnder(t *testing.T) {
+	sourceRoot := filepath.Join(t.TempDir(), "native")
+	root := t.TempDir()
+	var source, target string
+	var flags uintptr
+	err := mountNativeWritableUnder(
+		sourceRoot,
+		root,
+		2,
+		firecrackerproto.NativeWritableMountSpec{Target: "/var/lib/docker"},
+		func(gotSource, gotTarget, fsType string, gotFlags uintptr, data string) error {
+			source = gotSource
+			target = gotTarget
+			flags = gotFlags
+			if fsType != "" || data != "" {
+				t.Fatalf("bind mount arguments = %q, %q", fsType, data)
+			}
+			return nil
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if source != filepath.Join(sourceRoot, "2") ||
+		target != filepath.Join(root, "var/lib/docker") ||
+		flags != unix.MS_BIND {
+		t.Fatalf("bind mount = %q, %q, %#x", source, target, flags)
+	}
+	for _, path := range []string{source, target} {
+		info, err := os.Stat(path)
+		if err != nil || !info.IsDir() {
+			t.Fatalf("native writable directory %q = %+v, %v", path, info, err)
+		}
+	}
+}
+
+func TestMountNativeWritableUnderRejectsSymlinkTarget(t *testing.T) {
+	sourceRoot := filepath.Join(t.TempDir(), "native")
+	root := t.TempDir()
+	outside := t.TempDir()
+	if err := os.Symlink(outside, filepath.Join(root, "var")); err != nil {
+		t.Fatal(err)
+	}
+	mounted := false
+	err := mountNativeWritableUnder(
+		sourceRoot,
+		root,
+		0,
+		firecrackerproto.NativeWritableMountSpec{Target: "/var/lib/docker"},
+		func(string, string, string, uintptr, string) error {
+			mounted = true
+			return nil
+		},
+	)
+	if err == nil || !strings.Contains(err.Error(), "traverses symlink") {
+		t.Fatalf("symlink error = %v", err)
+	}
+	if mounted {
+		t.Fatal("native writable mount was attempted for a symlink target")
+	}
+}
+
 func TestPrepareContainerFileReplacesFinalSymlink(t *testing.T) {
 	root := t.TempDir()
 	if err := os.Mkdir(filepath.Join(root, "etc"), 0755); err != nil {

cmd/sbox/start.go

@@ -55,6 +55,10 @@ var StartCmd = cli.Command{
 			Name:  "enable-kvm",
 			Usage: "expose the configured KVM device in a runc sandbox",
 		},
+		cli.StringFlag{
+			Name:  "extra-config",
+			Usage: "runtime-specific configuration as a JSON object",
+		},
 		cli.StringFlag{
 			Name:  "cwd",
 			Usage: "working directory inside the sandbox",
@@ -139,7 +143,10 @@ var StartCmd = cli.Command{
 		if err != nil {
 			return err
 		}
-		extraConfig, err := startExtraConfig(context.Bool("enable-kvm"))
+		extraConfig, err := startExtraConfig(
+			context.String("extra-config"),
+			context.Bool("enable-kvm"),
+		)
 		if err != nil {
 			return err
 		}
@@ -209,13 +216,23 @@ func startRootfs(localPath, imageURL string, readonly bool) (*runtime.RootfsConf
 	return rootfs, nil
 }
 
-func startExtraConfig(enableKVM bool) (string, error) {
-	if !enableKVM {
+func startExtraConfig(raw string, enableKVM bool) (string, error) {
+	if strings.TrimSpace(raw) == "" && !enableKVM {
 		return "", nil
 	}
-	data, err := json.Marshal(struct {
-		EnableKVM bool `json:"enableKVM"`
-	}{EnableKVM: true})
+	config := make(map[string]json.RawMessage)
+	if strings.TrimSpace(raw) != "" {
+		if err := json.Unmarshal([]byte(raw), &config); err != nil {
+			return "", fmt.Errorf("decode --extra-config: %w", err)
+		}
+		if config == nil {
+			return "", fmt.Errorf("--extra-config must be a JSON object")
+		}
+	}
+	if enableKVM {
+		config["enableKVM"] = json.RawMessage("true")
+	}
+	data, err := json.Marshal(config)
 	if err != nil {
 		return "", err
 	}

cmd/sbox/start_test.go

@@ -62,13 +62,38 @@ func TestStorageMBToBytes(t *testing.T) {
 }
 
 func TestStartExtraConfig(t *testing.T) {
-	value, err := startExtraConfig(false)
+	value, err := startExtraConfig("", false)
 	require.NoError(t, err)
 	assert.Empty(t, value)
 
-	value, err = startExtraConfig(true)
+	value, err = startExtraConfig("", true)
 	require.NoError(t, err)
 	assert.JSONEq(t, `{"enableKVM":true}`, value)
+
+	value, err = startExtraConfig(
+		`{"nativeWritableMounts":[{"target":"/var/lib/docker"}]}`,
+		false,
+	)
+	require.NoError(t, err)
+	assert.JSONEq(t,
+		`{"nativeWritableMounts":[{"target":"/var/lib/docker"}]}`,
+		value,
+	)
+
+	value, err = startExtraConfig(
+		`{"enableKVM":false,"runtimeOption":"kept"}`,
+		true,
+	)
+	require.NoError(t, err)
+	assert.JSONEq(t,
+		`{"enableKVM":true,"runtimeOption":"kept"}`,
+		value,
+	)
+
+	for _, invalid := range []string{"null", "[]", "not-json"} {
+		_, err = startExtraConfig(invalid, false)
+		require.Error(t, err)
+	}
 }
 
 func TestStartRootfs(t *testing.T) {

doc/checkpoint-restore.md

@@ -214,6 +214,13 @@ image. The artifact overlay must not be used as the restored VM's writable
 image: checkpoint generations are immutable, the source may keep running, and
 concurrent restores require independent writable layers.
 
+Firecracker native writable mounts do not add checkpoint components. Their
+directories reside in the same `overlay.ext4` as the root overlay's upper and
+work directories, while the VM snapshot preserves the guest bind mounts. A
+checkpoint and restore therefore carries their data and mount state through
+the existing `overlay.ext4`, `memory`, and `vmstate` artifacts and applies the
+same quota, clone, durability, and storage-placement behavior.
+
 sandboxd uses `FICLONE` for these copies when possible:
 
 - the live writable image under

doc/runtime.md

@@ -102,6 +102,28 @@ and mount setup. An explicit writable-layer limit sizes this image, with a
 16 MiB minimum. Without an explicit limit, `default_overlay_size_bytes`
 applies and defaults to 10 GiB. The image is removed on sandbox deletion.
 
+A Firecracker start may expose directories from that same private ext4 image
+at selected guest paths. This is useful for workloads such as Docker that need
+a native filesystem instead of placing their own overlay on sandboxd's root
+OverlayFS. Configure the paths through the runtime-specific start configuration:
+
+```json
+{
+  "nativeWritableMounts": [
+    {"target": "/var/lib/docker"}
+  ]
+}
+```
+
+Each target is backed by a distinct, root-owned directory next to the root
+overlay's `upper` and `work` directories and is bind-mounted into the guest.
+It is not a host bind mount or an additional Firecracker drive. The root
+overlay and every native writable mount therefore share the single ext4 image
+and its writable-layer quota. Targets must be canonical absolute directory
+paths, may not overlap each other, an ordinary mount, or the guest's `/dev`,
+`/proc`, `/run`, `/sys`, and `/tmp` system mounts. At most 16 targets may be
+requested.
+
 EROFS and `rofs` mounts must also name regular EROFS image files and are
 attached as read-only drives. Read-only regular files are injected into the
 guest, limited to 1 MiB per file and 4 MiB in total; this narrow path supports
@@ -113,6 +135,6 @@ instead of being silently weakened.
 Private tmpfs mounts are supported with a bounded set of standard security,
 ownership, mode, inode, and size options.
 
-The private ext4 image remains in the filestore across sandboxd restart so the
-handler can recover the running VMM. It is cleaned by normal or idempotent
-sandbox deletion.
+The private ext4 image, including native writable mount data, remains in the
+filestore across sandboxd restart so the handler can recover the running VMM.
+It is cleaned by normal or idempotent sandbox deletion.

internal/firecrackerproto/protocol.go

@@ -104,6 +104,12 @@ type MountSpec struct {
 	Options []string `json:"options,omitempty"`
 }
 
+// NativeWritableMountSpec exposes a directory from the sandbox's private
+// ext4 writable layer at a distinct path inside the guest root filesystem.
+type NativeWritableMountSpec struct {
+	Target string `json:"target"`
+}
+
 type FileSpec struct {
 	Target   string `json:"target"`
 	Content  []byte `json:"content"`
@@ -112,14 +118,15 @@ type FileSpec struct {
 }
 
 type ConfigureRequest struct {
-	Hostname      string      `json:"hostname"`
-	RootDevice    string      `json:"root_device"`
-	OverlayDevice string      `json:"overlay_device"`
-	RootReadonly  bool        `json:"root_readonly,omitempty"`
-	Process       ProcessSpec `json:"process"`
-	Network       NetworkSpec `json:"network"`
-	Mounts        []MountSpec `json:"mounts,omitempty"`
-	Files         []FileSpec  `json:"files,omitempty"`
+	Hostname             string                    `json:"hostname"`
+	RootDevice           string                    `json:"root_device"`
+	OverlayDevice        string                    `json:"overlay_device"`
+	RootReadonly         bool                      `json:"root_readonly,omitempty"`
+	Process              ProcessSpec               `json:"process"`
+	Network              NetworkSpec               `json:"network"`
+	Mounts               []MountSpec               `json:"mounts,omitempty"`
+	NativeWritableMounts []NativeWritableMountSpec `json:"native_writable_mounts,omitempty"`
+	Files                []FileSpec                `json:"files,omitempty"`
 }
 
 type ExecRequest struct {

internal/firecrackerproto/protocol_test.go

@@ -64,6 +64,33 @@ func TestMessageRoundTripWithShortWrites(t *testing.T) {
 	}
 }
 
+func TestConfigureRequestRoundTripIncludesNativeWritableMounts(t *testing.T) {
+	var encoded bytes.Buffer
+	request := ConfigureRequest{
+		NativeWritableMounts: []NativeWritableMountSpec{{
+			Target: "/var/lib/docker",
+		}},
+	}
+	if err := WriteMessage(&encoded, MessageConfigure, request); err != nil {
+		t.Fatal(err)
+	}
+	messageType, payload, err := ReadMessage(&encoded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if messageType != MessageConfigure {
+		t.Fatalf("message type = %d, want %d", messageType, MessageConfigure)
+	}
+	var decoded ConfigureRequest
+	if err := Decode(payload, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded.NativeWritableMounts) != 1 ||
+		decoded.NativeWritableMounts[0].Target != "/var/lib/docker" {
+		t.Fatalf("decoded request = %+v", decoded)
+	}
+}
+
 func TestDecodeRejectsUnknownFields(t *testing.T) {
 	var request ExecRequest
 	if err := Decode([]byte(`{"command":"true","unexpected":1}`), &request); err == nil {

internal/server/server.go

@@ -1545,6 +1545,7 @@ func (h *sandboxService) Start(ctx context.Context, request *runtime.StartReques
 		DisableCgroup:           h.config.DisableCgroup,
 		SpecUpdates:             specUpdates,
 		WritableLayerLimitBytes: startReq.WritableLayerLimitBytes,
+		ExtraConfig:             startReq.ExtraConfig,
 		EnableKVM:               extraConfig.EnableKVM,
 		CheckpointDir:           checkpointDir,
 	}