Add tar index mode to erofs snapshotter

docs/snapshotters/erofs.md

@@ -173,6 +173,28 @@ When enabled via `enable_fsverity = true`, the snapshotter will:
 - Verify fsverity status before mounting layers
 - Skip fsverity if the filesystem or kernel does not support it
 
+## Tar Index Mode
+
+The EROFS differ also supports a "tar index" mode that offers a unique approach to handling OCI image layers:
+
+Instead of extracting the entire tar archive to create an EROFS filesystem, the tar index mode:
+1. Generates a tar index for the tar content
+2. Appends the original tar content to the index
+3. Creates a combined file: `[Tar index][Original tar content]`
+
+The tar index can be stored in a registry alongside image layers, allowing nodes to fetch it directly when needed. Typically, the tar index is much smaller than a full EROFS blob, making it more efficient to store and transfer. If the tar index is not available in the registry, it can be generated on the node as a fallback. When integrating with dm-verity, the registry can also store the dm-verity Merkle tree and root hash signature together with the tar index, enabling nodes to retrieve all necessary artifacts without redundant computation.
+
+In addition, we have a tar diffID for each layer according to the OCI image spec, so we don't need to reinvent a new way to verify the image layer content for confidential containers but just calculate the sha256 of the original tar data (because erofs could just reuse the tar data with 512-byte fs block size and build a minimal index for direct mounting of tar) out of the tar index mode in the guest and compare it with each diffID.
+
+### Configuration
+
+For the EROFS differ:
+
+```toml
+[plugins."io.containerd.differ.v1.erofs"]
+  enable_tar_index = true
+```
+
 ## TODO
 
 The EROFS Fsmerge feature is NOT supported in the current implementation

internal/erofsutils/mount_linux.go

@@ -17,6 +17,7 @@
 package erofsutils
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -46,6 +47,61 @@ func ConvertTarErofs(ctx context.Context, r io.Reader, layerPath, uuid string, m
 	return nil
 }
 
+// GenerateTarIndexAndAppendTar calculates tar index using --tar=i option
+// and appends the original tar content to create a combined EROFS layer.
+//
+// The `--tar=i` option instructs mkfs.erofs to only generate the tar index
+// for the tar content. The resulting file structure is:
+// [Tar index][Original tar content]
+func GenerateTarIndexAndAppendTar(ctx context.Context, r io.Reader, layerPath string, mkfsExtraOpts []string) error {
+	// Create a temporary file for storing the tar content
+	tarFile, err := os.CreateTemp("", "erofs-tar-*")
+	if err != nil {
+		return fmt.Errorf("failed to create temporary tar file: %w", err)
+	}
+	defer os.Remove(tarFile.Name())
+	defer tarFile.Close()
+
+	// Use TeeReader to process the input once while saving it to disk
+	teeReader := io.TeeReader(r, tarFile)
+
+	// Generate tar index directly to layerPath using --tar=i option
+	args := append([]string{"--tar=i", "--aufs", "--quiet"}, mkfsExtraOpts...)
+	args = append(args, layerPath)
+	cmd := exec.CommandContext(ctx, "mkfs.erofs", args...)
+	cmd.Stdin = teeReader
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("tar index generation failed with command 'mkfs.erofs %s': %s: %w",
+			strings.Join(args, " "), out, err)
+	}
+
+	// Log the command execution for debugging
+	log.G(ctx).Tracef("Generated tar index with command: %s %s, output: %s",
+		cmd.Path, strings.Join(cmd.Args, " "), string(out))
+
+	// Open layerPath for appending
+	f, err := os.OpenFile(layerPath, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open layer file for appending: %w", err)
+	}
+	defer f.Close()
+
+	// Rewind the temporary file
+	if _, err := tarFile.Seek(0, 0); err != nil {
+		return fmt.Errorf("failed to seek to the beginning of tar file: %w", err)
+	}
+
+	// Append tar content
+	if _, err := io.Copy(f, tarFile); err != nil {
+		return fmt.Errorf("failed to append tar to layer: %w", err)
+	}
+
+	log.G(ctx).Infof("Successfully generated EROFS layer with tar index and tar content: %s", layerPath)
+
+	return nil
+}
+
 func ConvertErofs(ctx context.Context, layerPath string, srcDir string, mkfsExtraOpts []string) error {
 	args := append([]string{"--quiet", "-Enoinline_data"}, mkfsExtraOpts...)
 	args = append(args, layerPath, srcDir)
@@ -99,3 +155,15 @@ func MountsToLayer(mounts []mount.Mount) (string, error) {
 	}
 	return layer, nil
 }
+
+// SupportGenerateFromTar checks if the installed version of mkfs.erofs supports
+// the tar mode (--tar option).
+func SupportGenerateFromTar() (bool, error) {
+	cmd := exec.Command("mkfs.erofs", "--help")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return false, fmt.Errorf("failed to run mkfs.erofs --help: %w", err)
+	}
+
+	return bytes.Contains(output, []byte("--tar=")), nil
+}

plugins/diff/erofs/differ_linux.go

@@ -58,15 +58,42 @@ type differ interface {
 type erofsDiff struct {
 	store         content.Store
 	mkfsExtraOpts []string
+	// enableTarIndex enables generating tar index for tar content
+	// instead of fully converting the tar to EROFS format
+	enableTarIndex bool
 }
 
-func NewErofsDiffer(store content.Store, mkfsExtraOpts []string) differ {
-	return &erofsDiff{
-		store:         store,
-		mkfsExtraOpts: mkfsExtraOpts,
+// DifferOpt is an option for configuring the erofs differ
+type DifferOpt func(d *erofsDiff)
+
+// WithMkfsOptions sets extra options for mkfs.erofs
+func WithMkfsOptions(opts []string) DifferOpt {
+	return func(d *erofsDiff) {
+		d.mkfsExtraOpts = opts
 	}
 }
 
+// WithTarIndexMode enables tar index mode for EROFS layers
+func WithTarIndexMode() DifferOpt {
+	return func(d *erofsDiff) {
+		d.enableTarIndex = true
+	}
+}
+
+// NewErofsDiffer creates a new EROFS differ with the provided options
+func NewErofsDiffer(store content.Store, opts ...DifferOpt) differ {
+	d := &erofsDiff{
+		store: store,
+	}
+
+	// Apply all options
+	for _, opt := range opts {
+		opt(d)
+	}
+
+	return d
+}
+
 // A valid EROFS native layer media type should end with ".erofs".
 //
 // Please avoid using any +suffix to list the algorithms used inside EROFS
@@ -301,10 +328,22 @@ func (s erofsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []
 		r: io.TeeReader(processor, digester.Hash()),
 	}
 
-	u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+desc.Digest))
-	err = erofsutils.ConvertTarErofs(ctx, rc, layerBlobPath, u.String(), s.mkfsExtraOpts)
-	if err != nil {
-		return emptyDesc, fmt.Errorf("failed to convert erofs: %w", err)
+	// Choose between tar index or tar conversion mode
+	if s.enableTarIndex {
+		// Use the tar index method: generate tar index and append tar
+		err = erofsutils.GenerateTarIndexAndAppendTar(ctx, rc, layerBlobPath, s.mkfsExtraOpts)
+		if err != nil {
+			return emptyDesc, fmt.Errorf("failed to generate tar index: %w", err)
+		}
+		log.G(ctx).WithField("path", layerBlobPath).Debug("Applied layer using tar index mode")
+	} else {
+		// Use the tar method: fully convert tar to EROFS
+		u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+desc.Digest))
+		err = erofsutils.ConvertTarErofs(ctx, rc, layerBlobPath, u.String(), s.mkfsExtraOpts)
+		if err != nil {
+			return emptyDesc, fmt.Errorf("failed to convert tar to erofs: %w", err)
+		}
+		log.G(ctx).WithField("path", layerBlobPath).Debug("Applied layer using tar conversion mode")
 	}
 
 	// Read any trailing data

plugins/diff/erofs/plugin/plugin_linux.go

@@ -18,9 +18,9 @@ package plugin
 
 import (
 	"fmt"
-	"os/exec"
 
 	"github.com/containerd/containerd/v2/core/metadata"
+	"github.com/containerd/containerd/v2/internal/erofsutils"
 	"github.com/containerd/containerd/v2/plugins"
 	"github.com/containerd/containerd/v2/plugins/diff/erofs"
 	"github.com/containerd/platforms"
@@ -32,6 +32,10 @@ import (
 type Config struct {
 	// MkfsOptions are extra options used for the applier
 	MkfsOptions []string `toml:"mkfs_options"`
+
+	// EnableTarIndex enables the tar index mode where the index is generated
+	// for tar content without extracting the tar
+	EnableTarIndex bool `toml:"enable_tar_index"`
 }
 
 func init() {
@@ -43,9 +47,12 @@ func init() {
 		},
 		Config: &Config{},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-			_, err := exec.LookPath("mkfs.erofs")
+			tarModeSupported, err := erofsutils.SupportGenerateFromTar()
 			if err != nil {
-				return nil, fmt.Errorf("could not find mkfs.erofs: %v: %w", err, plugin.ErrSkipPlugin)
+				return nil, fmt.Errorf("failed to check mkfs.erofs availability: %v: %w", err, plugin.ErrSkipPlugin)
+			}
+			if !tarModeSupported {
+				return nil, fmt.Errorf("mkfs.erofs does not support tar mode (--tar option), disabling erofs differ: %w", plugin.ErrSkipPlugin)
 			}
 
 			md, err := ic.GetSingle(plugins.MetadataPlugin)
@@ -57,7 +64,17 @@ func init() {
 			cs := md.(*metadata.DB).ContentStore()
 			config := ic.Config.(*Config)
 
-			return erofs.NewErofsDiffer(cs, config.MkfsOptions), nil
+			var opts []erofs.DifferOpt
+
+			if len(config.MkfsOptions) > 0 {
+				opts = append(opts, erofs.WithMkfsOptions(config.MkfsOptions))
+			}
+
+			if config.EnableTarIndex {
+				opts = append(opts, erofs.WithTarIndexMode())
+			}
+
+			return erofs.NewErofsDiffer(cs, opts...), nil
 		},
 	})
 }