Merge branch 'main' into bump-lifecycle

acceptance/acceptance_test.go

@@ -2253,6 +2253,92 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 					})
 				})
+
+				when("build --buildpack <flattened buildpack>", func() {
+					var (
+						tmpDir                         string
+						flattenedPackageName           string
+						simplePackageConfigFixtureName = "package.toml"
+					)
+
+					generateAggregatePackageToml := func(buildpackURI, nestedPackageName, operatingSystem string) string {
+						t.Helper()
+						packageTomlFile, err := os.CreateTemp(tmpDir, "package_aggregate-*.toml")
+						assert.Nil(err)
+
+						pack.FixtureManager().TemplateFixtureToFile(
+							"package_aggregate.toml",
+							packageTomlFile,
+							map[string]interface{}{
+								"BuildpackURI": buildpackURI,
+								"PackageName":  nestedPackageName,
+								"OS":           operatingSystem,
+							},
+						)
+
+						assert.Nil(packageTomlFile.Close())
+						return packageTomlFile.Name()
+					}
+
+					it.Before(func() {
+						h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
+						h.SkipIf(t, imageManager.HostOS() == "windows", "buildpack directories not supported on windows")
+
+						var err error
+						tmpDir, err = os.MkdirTemp("", "buildpack-package-flattened-tests")
+						assert.Nil(err)
+
+						buildpackManager = buildpacks.NewBuildModuleManager(t, assert)
+						buildpackManager.PrepareBuildModules(tmpDir, buildpacks.BpSimpleLayersParent, buildpacks.BpSimpleLayers)
+
+						// set up a flattened buildpack
+						packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
+						nestedPackageName := "test/flattened-package-" + h.RandString(10)
+						nestedPackage := buildpacks.NewPackageImage(
+							t,
+							pack,
+							nestedPackageName,
+							packageTomlPath,
+							buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
+						)
+						buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
+						assertImage.ExistsLocally(nestedPackageName)
+
+						aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
+						flattenedPackageName = "test/package-" + h.RandString(10)
+
+						_ = pack.RunSuccessfully(
+							"buildpack", "package", flattenedPackageName,
+							"-c", aggregatePackageToml,
+							"--flatten",
+						)
+
+						assertImage.ExistsLocally(flattenedPackageName)
+						assertImage.HasLengthLayers(flattenedPackageName, 1)
+					})
+
+					it.After(func() {
+						assert.Nil(os.RemoveAll(tmpDir))
+						imageManager.CleanupImages(flattenedPackageName)
+					})
+
+					when("--flatten", func() {
+						it("does not write duplicate tar files when creating the ephemeral builder", func() {
+							output := pack.RunSuccessfully(
+								"build", repoName,
+								"-p", filepath.Join("testdata", "mock_app"),
+								"--buildpack", fmt.Sprintf("docker://%s", flattenedPackageName),
+								"--builder", builderName,
+							)
+							// buildpack returning an empty tar file is non-deterministic,
+							// but we expect one of them to throw the warning
+							h.AssertContainsMatch(t, output, "Buildpack '(simple/layers@simple-layers-version|simple/layers/parent@simple-layers-parent-version)' is a component of a flattened buildpack that will be added elsewhere, skipping...")
+
+							// simple/layers BP exists on the builder and in the flattened buildpack
+							h.AssertContainsMatch(t, output, "Buildpack 'simple/layers@simple-layers-version' already exists on builder with same contents, skipping...")
+						})
+					})
+				})
 			})
 
 			when("inspecting builder", func() {

acceptance/assertions/image.go

@@ -69,6 +69,13 @@ func (a ImageAssertionManager) HasLabelWithData(image, label, data string) {
 	a.assert.Contains(label, data)
 }
 
+func (a ImageAssertionManager) HasLengthLayers(image string, length int) {
+	a.testObject.Helper()
+	inspect, err := a.imageManager.InspectLocal(image)
+	a.assert.Nil(err)
+	a.assert.TrueWithMessage(len(inspect.RootFS.Layers) == length, fmt.Sprintf("expected image to have %d layers, found %d", length, len(inspect.RootFS.Layers)))
+}
+
 func (a ImageAssertionManager) RunsWithOutput(image string, expectedOutputs ...string) {
 	a.testObject.Helper()
 	containerName := "test-" + h.RandString(10)

acceptance/invoke/pack.go

@@ -233,6 +233,7 @@ const (
 	RunImageExtensions
 	StackValidation
 	ForceRebase
+	BuildpackFlatten
 )
 
 var featureTests = map[Feature]func(i *PackInvoker) bool{
@@ -254,6 +255,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	ForceRebase: func(i *PackInvoker) bool {
 		return i.atLeast("v0.30.0")
 	},
+	BuildpackFlatten: func(i *PackInvoker) bool {
+		return i.atLeast("v0.30.0")
+	},
 }
 
 func (i *PackInvoker) SupportsFeature(f Feature) bool {

go.mod

@@ -29,12 +29,12 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sclevine/spec v1.4.0
 	github.com/spf13/cobra v1.7.0
-	golang.org/x/crypto v0.10.0
-	golang.org/x/mod v0.10.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/mod v0.12.0
 	golang.org/x/oauth2 v0.9.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.9.0
-	golang.org/x/text v0.10.0
+	golang.org/x/term v0.10.0
+	golang.org/x/text v0.11.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -345,15 +345,15 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -416,17 +416,17 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

internal/build/container_ops.go

@@ -207,92 +207,53 @@ func findMount(info types.ContainerJSON, dst string) (types.MountPoint, error) {
 	return types.MountPoint{}, fmt.Errorf("no matching mount found for %s", dst)
 }
 
-// WriteProjectMetadata
-func WriteProjectMetadata(p string, metadata files.ProjectMetadata, os string) ContainerOperation {
-	return func(ctrClient DockerClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
-		buf := &bytes.Buffer{}
-		err := toml.NewEncoder(buf).Encode(metadata)
-		if err != nil {
-			return errors.Wrap(err, "marshaling project metadata")
-		}
+func writeToml(ctrClient DockerClient, ctx context.Context, data interface{}, dstPath string, containerID string, os string, stdout, stderr io.Writer) error {
+	buf := &bytes.Buffer{}
+	err := toml.NewEncoder(buf).Encode(data)
+	if err != nil {
+		return errors.Wrapf(err, "marshaling data to %s", dstPath)
+	}
 
-		tarBuilder := archive.TarBuilder{}
+	tarBuilder := archive.TarBuilder{}
 
-		tarPath := p
-		if os == "windows" {
-			tarPath = paths.WindowsToSlash(p)
-		}
+	tarPath := dstPath
+	if os == "windows" {
+		tarPath = paths.WindowsToSlash(dstPath)
+	}
 
-		tarBuilder.AddFile(tarPath, 0755, archive.NormalizedDateTime, buf.Bytes())
-		reader := tarBuilder.Reader(archive.DefaultTarWriterFactory())
-		defer reader.Close()
+	tarBuilder.AddFile(tarPath, 0755, archive.NormalizedDateTime, buf.Bytes())
+	reader := tarBuilder.Reader(archive.DefaultTarWriterFactory())
+	defer reader.Close()
 
-		if os == "windows" {
-			dirName := paths.WindowsDir(p)
-			return copyDirWindows(ctx, ctrClient, containerID, reader, dirName, stdout, stderr)
-		}
+	if os == "windows" {
+		dirName := paths.WindowsDir(dstPath)
+		return copyDirWindows(ctx, ctrClient, containerID, reader, dirName, stdout, stderr)
+	}
 
-		return ctrClient.CopyToContainer(ctx, containerID, "/", reader, types.CopyToContainerOptions{})
+	return ctrClient.CopyToContainer(ctx, containerID, "/", reader, types.CopyToContainerOptions{})
+}
+
+// WriteProjectMetadata writes a `project-metadata.toml` based on the ProjectMetadata provided to the destination path.
+func WriteProjectMetadata(dstPath string, metadata files.ProjectMetadata, os string) ContainerOperation {
+	return func(ctrClient DockerClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
+		return writeToml(ctrClient, ctx, metadata, dstPath, containerID, os, stdout, stderr)
 	}
 }
 
 // WriteStackToml writes a `stack.toml` based on the StackMetadata provided to the destination path.
 func WriteStackToml(dstPath string, stack builder.StackMetadata, os string) ContainerOperation {
 	return func(ctrClient DockerClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
-		buf := &bytes.Buffer{}
-		err := toml.NewEncoder(buf).Encode(stack)
-		if err != nil {
-			return errors.Wrap(err, "marshaling stack metadata")
-		}
-
-		tarBuilder := archive.TarBuilder{}
-
-		tarPath := dstPath
-		if os == "windows" {
-			tarPath = paths.WindowsToSlash(dstPath)
-		}
-
-		tarBuilder.AddFile(tarPath, 0755, archive.NormalizedDateTime, buf.Bytes())
-		reader := tarBuilder.Reader(archive.DefaultTarWriterFactory())
-		defer reader.Close()
-
-		if os == "windows" {
-			dirName := paths.WindowsDir(dstPath)
-			return copyDirWindows(ctx, ctrClient, containerID, reader, dirName, stdout, stderr)
-		}
-
-		return ctrClient.CopyToContainer(ctx, containerID, "/", reader, types.CopyToContainerOptions{})
+		return writeToml(ctrClient, ctx, stack, dstPath, containerID, os, stdout, stderr)
 	}
 }
 
 // WriteRunToml writes a `run.toml` based on the RunConfig provided to the destination path.
 func WriteRunToml(dstPath string, runImages []builder.RunImageMetadata, os string) ContainerOperation {
+	runImageData := builder.RunImages{
+		Images: runImages,
+	}
 	return func(ctrClient DockerClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
-		buf := &bytes.Buffer{}
-		err := toml.NewEncoder(buf).Encode(builder.RunImages{
-			Images: runImages,
-		})
-		if err != nil {
-			return errors.Wrap(err, "marshaling run metadata")
-		}
-
-		tarBuilder := archive.TarBuilder{}
-
-		tarPath := dstPath
-		if os == "windows" {
-			tarPath = paths.WindowsToSlash(dstPath)
-		}
-
-		tarBuilder.AddFile(tarPath, 0755, archive.NormalizedDateTime, buf.Bytes())
-		reader := tarBuilder.Reader(archive.DefaultTarWriterFactory())
-		defer reader.Close()
-
-		if os == "windows" {
-			dirName := paths.WindowsDir(dstPath)
-			return copyDirWindows(ctx, ctrClient, containerID, reader, dirName, stdout, stderr)
-		}
-
-		return ctrClient.CopyToContainer(ctx, containerID, "/", reader, types.CopyToContainerOptions{})
+		return writeToml(ctrClient, ctx, runImageData, dstPath, containerID, os, stdout, stderr)
 	}
 }
 

internal/build/container_ops_test.go

@@ -638,6 +638,7 @@ drwxrwxrwx    2 123      456 (.*) some-vol
 `)
 		})
 	})
+
 	when("#EnsureVolumeAccess", func() {
 		it("changes owner of volume", func() {
 			h.SkipIf(t, osType != "windows", "no-op for linux")

internal/build/lifecycle_execution.go

@@ -469,13 +469,17 @@ func (l *LifecycleExecution) Restore(ctx context.Context, buildCache Cache, kani
 	// for kaniko
 	kanikoCacheBindOp := NullOp()
 	if (l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild()) ||
-		(l.platformAPI.AtLeast("0.12") && (l.hasExtensionsForBuild() || l.hasExtensionsForRun())) {
+		l.platformAPI.AtLeast("0.12") {
 		if l.hasExtensionsForBuild() {
 			flags = append(flags, "-build-image", l.opts.BuilderImage)
 			registryImages = append(registryImages, l.opts.BuilderImage)
 		}
-
-		kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir()))
+		if l.runImageChanged() || l.hasExtensionsForRun() {
+			registryImages = append(registryImages, l.runImageAfterExtensions())
+		}
+		if l.hasExtensionsForBuild() || l.hasExtensionsForRun() {
+			kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir()))
+		}
 	}
 
 	// for auths
@@ -842,9 +846,12 @@ func (l *LifecycleExecution) hasExtensionsForBuild() bool {
 func (l *LifecycleExecution) hasExtensionsForRun() bool {
 	var amd files.Analyzed
 	if _, err := toml.DecodeFile(filepath.Join(l.tmpDir, "analyzed.toml"), &amd); err != nil {
+		l.logger.Warnf("failed to parse analyzed.toml file, assuming no run image extensions: %s", err)
 		return false
 	}
 	if amd.RunImage == nil {
+		// this shouldn't be reachable
+		l.logger.Warnf("found no run image in analyzed.toml file, assuming no run image extensions...")
 		return false
 	}
 	return amd.RunImage.Extend
@@ -853,15 +860,22 @@ func (l *LifecycleExecution) hasExtensionsForRun() bool {
 func (l *LifecycleExecution) runImageAfterExtensions() string {
 	var amd files.Analyzed
 	if _, err := toml.DecodeFile(filepath.Join(l.tmpDir, "analyzed.toml"), &amd); err != nil {
+		l.logger.Warnf("failed to parse analyzed.toml file, assuming run image did not change: %s", err)
 		return l.opts.RunImage
 	}
 	if amd.RunImage == nil || amd.RunImage.Image == "" {
 		// this shouldn't be reachable
+		l.logger.Warnf("found no run image in analyzed.toml file, assuming run image did not change...")
 		return l.opts.RunImage
 	}
 	return amd.RunImage.Image
 }
 
+func (l *LifecycleExecution) runImageChanged() bool {
+	currentRunImage := l.runImageAfterExtensions()
+	return currentRunImage != "" && currentRunImage != l.opts.RunImage
+}
+
 func (l *LifecycleExecution) appendLayoutOperations(opts []PhaseConfigProviderOperation) ([]PhaseConfigProviderOperation, error) {
 	layoutDir := filepath.Join(paths.RootDir, "layout-repo")
 	opts = append(opts, WithEnv("CNB_USE_LAYOUT=true", "CNB_LAYOUT_DIR="+layoutDir, "CNB_EXPERIMENTAL_MODE=warn"))