XX

Split copy pipeline up

Signed-off-by: Miloslav Trmač <[email protected]>

Author: mtrmac

copy/compression.go

@@ -104,16 +104,17 @@ func (ic *imageCopier) bpcPreserveEncrypted(stream *sourceStream, _ bpDetectComp
 
 // bpcCompressUncompressed checks if we should be compressing an uncompressed input, and returns a *bpCompressionStepData if so.
 func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
-	if ic.c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed {
+	var chosenFormat *compressiontypes.Algorithm
+	if ic.c.compressionFormat != nil {
+		chosenFormat = ic.c.compressionFormat
+	} else {
+		chosenFormat = defaultCompressionFormat
+	}
+	if ic.c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed &&
+		ic.src.CanCompressLayer(stream.info.MediaType, chosenFormat) {
 		logrus.Debugf("Compressing blob on the fly")
-		var uploadedAlgorithm *compressiontypes.Algorithm
-		if ic.c.compressionFormat != nil {
-			uploadedAlgorithm = ic.c.compressionFormat
-		} else {
-			uploadedAlgorithm = defaultCompressionFormat
-		}
 
-		reader, annotations := ic.c.compressedStream(stream.reader, *uploadedAlgorithm)
+		reader, annotations := ic.c.compressedStream(stream.reader, *chosenFormat)
 		// Note: reader must be closed on all return paths.
 		stream.reader = reader
 		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
@@ -122,10 +123,10 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp
 		}
 		return &bpCompressionStepData{
 			operation:              types.Compress,
-			uploadedAlgorithm:      uploadedAlgorithm,
+			uploadedAlgorithm:      chosenFormat,
 			uploadedAnnotations:    annotations,
 			srcCompressorName:      detected.srcCompressorName,
-			uploadedCompressorName: uploadedAlgorithm.Name(),
+			uploadedCompressorName: chosenFormat.Name(),
 			closers:                []io.Closer{reader},
 		}, nil
 	}
@@ -135,7 +136,8 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp
 // bpcRecompressCompressed checks if we should be recompressing a compressed input to another format, and returns a *bpCompressionStepData if so.
 func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
 	if ic.c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed &&
-		ic.c.compressionFormat != nil && ic.c.compressionFormat.Name() != detected.format.Name() {
+		ic.c.compressionFormat != nil && ic.c.compressionFormat.Name() != detected.format.Name() &&
+		ic.src.CanCompressLayer(stream.info.MediaType, ic.c.compressionFormat) {
 		// When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally
 		// re-compressed using the desired format.
 		logrus.Debugf("Blob will be converted")
@@ -173,7 +175,8 @@ func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bp
 
 // bpcDecompressCompressed checks if we should be decompressing a compressed input, and returns a *bpCompressionStepData if so.
 func (ic *imageCopier) bpcDecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
-	if ic.c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed {
+	if ic.c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed &&
+		ic.src.CanDecompressLayer(stream.info.MediaType) {
 		logrus.Debugf("Blob will be decompressed")
 		s, err := detected.decompressor(stream.reader)
 		if err != nil {

internal/image/docker_schema1.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
+	compression "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -257,3 +258,18 @@ func (m *manifestSchema1) CanSubstituteLayers() bool {
 func (m *manifestSchema1) CanChangeLayerCompression(mimeType string) bool {
 	return true // There are no MIME types in the manifest, so we must assume a valid image.
 }
+
+// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestSchema1) CanDecompressLayer(mimeType string) bool {
+	return true // There are no MIME types in the manifest, so we must assume a valid image.
+}
+
+// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestSchema1) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool {
+	// There are no MIME types in the manifest, so we must assume a valid image. But gzip is the only supported format.
+	return algorithm.Name() == compression.GzipAlgorithmName
+}

internal/image/docker_schema2.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containers/image/v5/internal/iolimits"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/blobinfocache/none"
+	compression "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -413,3 +414,17 @@ func (m *manifestSchema2) CanSubstituteLayers() bool {
 func (m *manifestSchema2) CanChangeLayerCompression(mimeType string) bool {
 	return m.m.CanChangeLayerCompression(mimeType)
 }
+
+// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestSchema2) CanDecompressLayer(mimeType string) bool {
+	return m.m.CanDecompressLayer(mimeType)
+}
+
+// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestSchema2) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool {
+	return m.m.CanCompressLayer(mimeType, algorithm)
+}

internal/image/manifest.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
+	compression "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -57,7 +58,14 @@ type genericManifest interface {
 	// CanSubstituteBlobs returns true if the generic image copy code is allowed to substitute layers with their semantic equivalents.
 	CanSubstituteLayers() bool
 	// CanChangeLayerCompression returns true if it is allowed to compress/decompress layers with mimeType (and the code can handle that)
+	// FIXME FIXME: This does not say which compression formats are supported, and it almost certainly should.
 	CanChangeLayerCompression(mimeType string) bool
+	// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+	// FIXME FIXME: Some other API
+	CanDecompressLayer(mimeType string) bool
+	// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+	// FIXME FIXME: Some other API
+	CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool
 }
 
 // manifestInstanceFromBlob returns a genericManifest implementation for (manblob, mt) in src.

internal/image/oci.go

@@ -10,6 +10,7 @@ import (
 	internalManifest "github.com/containers/image/v5/internal/manifest"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/blobinfocache/none"
+	compression "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -272,3 +273,17 @@ func (m *manifestOCI1) CanSubstituteLayers() bool {
 func (m *manifestOCI1) CanChangeLayerCompression(mimeType string) bool {
 	return m.m.CanChangeLayerCompression(mimeType)
 }
+
+// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestOCI1) CanDecompressLayer(mimeType string) bool {
+	return m.m.CanDecompressLayer(mimeType)
+}
+
+// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *manifestOCI1) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool {
+	return m.m.CanCompressLayer(mimeType, algorithm)
+}

manifest/docker_schema2.go

@@ -301,3 +301,19 @@ func (m *Schema2) ImageID([]digest.Digest) (string, error) {
 func (m *Schema2) CanChangeLayerCompression(mimeType string) bool {
 	return compressionVariantsRecognizeMIMEType(schema2CompressionMIMETypeSets, mimeType)
 }
+
+// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *Schema2) CanDecompressLayer(mimeType string) bool {
+	_, err := compressionVariantMIMEType(schema2CompressionMIMETypeSets, mimeType, nil)
+	return err == nil
+}
+
+// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *Schema2) CanCompressLayer(mimeType string, algorithm *compressiontypes.Algorithm) bool {
+	_, err := compressionVariantMIMEType(schema2CompressionMIMETypeSets, mimeType, algorithm)
+	return err == nil
+}

manifest/oci.go

@@ -258,3 +258,25 @@ func (m *OCI1) CanChangeLayerCompression(mimeType string) bool {
 	}
 	return compressionVariantsRecognizeMIMEType(oci1CompressionMIMETypeSets, mimeType)
 }
+
+// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *OCI1) CanDecompressLayer(mimeType string) bool {
+	if m.Config.MediaType != imgspecv1.MediaTypeImageConfig {
+		return false
+	}
+	_, err := compressionVariantMIMEType(oci1CompressionMIMETypeSets, mimeType, nil)
+	return err == nil
+}
+
+// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that)
+// FIXME FIXME: Some other API
+// FIXME FIXME: Tests
+func (m *OCI1) CanCompressLayer(mimeType string, algorithm *compressiontypes.Algorithm) bool {
+	if m.Config.MediaType != imgspecv1.MediaTypeImageConfig {
+		return false
+	}
+	_, err := compressionVariantMIMEType(oci1CompressionMIMETypeSets, mimeType, algorithm)
+	return err == nil
+}