diff --git a/client/client_test.go b/client/client_test.go index 6c49b5e8c07d..541b285848ca 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -227,6 +227,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testSnapshotWithMultipleBlobs, testExportLocalNoPlatformSplit, testExportLocalNoPlatformSplitOverwrite, + testExportLocalForcePlatformSplit, testSolverOptLocalDirsStillWorks, testOCIIndexMediatype, testLayerLimitOnMounts, @@ -6951,6 +6952,62 @@ func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbo }, }, "", frontend, nil) require.Error(t, err) + require.ErrorContains(t, err, "cannot overwrite hello-linux from") + require.ErrorContains(t, err, "when split option is disabled") +} + +func testExportLocalForcePlatformSplit(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureMultiPlatform) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := llb.Scratch().File( + llb.Mkfile("foo", 0600, []byte("hello")), + ) + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + return c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + } + + destDir := t.TempDir() + _, err = c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + Attrs: map[string]string{ + "platform-split": "true", + }, + }, + }, + }, "", frontend, nil) + require.NoError(t, err) + + fis, err := os.ReadDir(destDir) + require.NoError(t, err) + + require.Len(t, fis, 1, "expected one files in the output directory") + + expPlatform := strings.ReplaceAll(platforms.FormatAll(platforms.DefaultSpec()), "/", "_") + _, err = os.Stat(filepath.Join(destDir, expPlatform+"/")) + require.NoError(t, err) + + fis, err = os.ReadDir(filepath.Join(destDir, expPlatform)) + require.NoError(t, err) + + require.Len(t, fis, 1, "expected one files in the output directory for platform "+expPlatform) + + dt, err := os.ReadFile(filepath.Join(destDir, expPlatform, "foo")) + require.NoError(t, err) + require.Equal(t, "hello", string(dt)) } func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) { diff --git a/exporter/local/export.go b/exporter/local/export.go index 29c6afbac990..10e3a87ce3a0 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -117,7 +117,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error { return func() error { - outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts) + outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, isMap, e.opts) if err != nil { return err } @@ -125,9 +125,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source defer cleanup() } - if !e.opts.PlatformSplit { + lbl := "copying files" + if !e.opts.UsePlatformSplit(isMap) { // check for duplicate paths - err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error { + err = fsWalk(ctx, outputFS, "", func(p string, entry os.DirEntry, err error) error { if entry.IsDir() { return nil } @@ -145,23 +146,18 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source if err != nil { return err } - } - - lbl := "copying files" - if isMap { + } else { lbl += " " + k - if e.opts.PlatformSplit { - st := &fstypes.Stat{ - Mode: uint32(os.ModeDir | 0755), - Path: strings.ReplaceAll(k, "/", "_"), - } - if e.opts.Epoch != nil { - st.ModTime = e.opts.Epoch.UnixNano() - } - outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) - if err != nil { - return err - } + st := &fstypes.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: strings.ReplaceAll(k, "/", "_"), + } + if e.opts.Epoch != nil { + st.ModTime = e.opts.Epoch.UnixNano() + } + outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) + if err != nil { + return err } } diff --git a/exporter/local/export_unix.go b/exporter/local/export_unix.go new file mode 100644 index 000000000000..d2612182fd38 --- /dev/null +++ b/exporter/local/export_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package local + +import ( + "context" + gofs "io/fs" + + "github.com/tonistiigi/fsutil" +) + +func fsWalk(ctx context.Context, fs fsutil.FS, s string, walkFn gofs.WalkDirFunc) error { + return fs.Walk(ctx, s, walkFn) +} diff --git a/exporter/local/export_windows.go b/exporter/local/export_windows.go new file mode 100644 index 000000000000..dea1821a71ad --- /dev/null +++ b/exporter/local/export_windows.go @@ -0,0 +1,17 @@ +package local + +import ( + "context" + gofs "io/fs" + + "github.com/Microsoft/go-winio" + "github.com/tonistiigi/fsutil" +) + +func fsWalk(ctx context.Context, fs fsutil.FS, s string, walkFn gofs.WalkDirFunc) error { + // Windows has some special files that require + // SeBackupPrivilege to be accessed. Ref #4994 + return winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error { + return fs.Walk(ctx, s, walkFn) + }) +} diff --git a/exporter/local/fs.go b/exporter/local/fs.go index 1985ed0096dc..6e68b0aa2a59 100644 --- a/exporter/local/fs.go +++ b/exporter/local/fs.go @@ -38,12 +38,18 @@ const ( type CreateFSOpts struct { Epoch *time.Time AttestationPrefix string - PlatformSplit bool + PlatformSplit *bool +} + +func (c *CreateFSOpts) UsePlatformSplit(isMap bool) bool { + if c.PlatformSplit == nil { + return isMap + } + return *c.PlatformSplit } func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { rest := make(map[string]string) - c.PlatformSplit = true var err error c.Epoch, opt, err = epoch.ParseExporterAttrs(opt) @@ -60,7 +66,7 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { if err != nil { return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v) } - c.PlatformSplit = b + c.PlatformSplit = &b default: rest[k] = v } @@ -69,7 +75,7 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) { return rest, nil } -func CreateFS(ctx context.Context, sessionID string, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, defaultTime time.Time, opt CreateFSOpts) (fsutil.FS, func() error, error) { +func CreateFS(ctx context.Context, sessionID string, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, defaultTime time.Time, isMap bool, opt CreateFSOpts) (fsutil.FS, func() error, error) { var cleanup func() error var src string var err error @@ -174,6 +180,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab return nil, nil, err } stmtFS := staticfs.NewFS() + split := opt.UsePlatformSplit(isMap) names := map[string]struct{}{} for i, stmt := range stmts { @@ -183,7 +190,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab } name := opt.AttestationPrefix + path.Base(attestations[i].Path) - if !opt.PlatformSplit { + if !split { nameExt := path.Ext(name) namBase := strings.TrimSuffix(name, nameExt) name = fmt.Sprintf("%s.%s%s", namBase, strings.ReplaceAll(k, "/", "_"), nameExt) diff --git a/exporter/tar/export.go b/exporter/tar/export.go index 770b38254d10..57d9eef10464 100644 --- a/exporter/tar/export.go +++ b/exporter/tar/export.go @@ -95,9 +95,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source } now := time.Now().Truncate(time.Second) + isMap := len(inp.Refs) > 0 getDir := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) (*fsutil.Dir, error) { - outputFS, cleanup, err := local.CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts) + outputFS, cleanup, err := local.CreateFS(ctx, sessionID, k, ref, attestations, now, isMap, e.opts) if err != nil { return nil, err } @@ -119,7 +120,6 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source }, nil } - isMap := len(inp.Refs) > 0 if _, ok := inp.Metadata[exptypes.ExporterPlatformsKey]; isMap && !ok { return nil, nil, errors.Errorf("unable to export multiple refs, missing platforms mapping") }