diff --git a/docker/docker_image_dest.go b/docker/docker_image_dest.go index 9761f58e84..906d79c10c 100644 --- a/docker/docker_image_dest.go +++ b/docker/docker_image_dest.go @@ -17,6 +17,21 @@ import ( "github.com/pkg/errors" ) +var manifestMIMETypes = []string{ + // TODO(runcom): we'll add OCI as part of another PR here + manifest.DockerV2Schema2MediaType, + manifest.DockerV2Schema1SignedMediaType, + manifest.DockerV2Schema1MediaType, +} + +func supportedManifestMIMETypesMap() map[string]bool { + m := make(map[string]bool, len(manifestMIMETypes)) + for _, mt := range manifestMIMETypes { + m[mt] = true + } + return m +} + type dockerImageDestination struct { ref dockerReference c *dockerClient @@ -47,12 +62,7 @@ func (d *dockerImageDestination) Close() { } func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { - return []string{ - // TODO(runcom): we'll add OCI as part of another PR here - manifest.DockerV2Schema2MediaType, - manifest.DockerV2Schema1SignedMediaType, - manifest.DockerV2Schema1MediaType, - } + return manifestMIMETypes } // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. diff --git a/docker/docker_image_src.go b/docker/docker_image_src.go index 563e6aa6c3..1526a9c105 100644 --- a/docker/docker_image_src.go +++ b/docker/docker_image_src.go @@ -39,6 +39,17 @@ func newImageSource(ctx *types.SystemContext, ref dockerReference, requestedMani if requestedManifestMIMETypes == nil { requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes } + supportedMIMEs := supportedManifestMIMETypesMap() + acceptableRequestedMIMEs := false + for _, mtrequested := range requestedManifestMIMETypes { + if supportedMIMEs[mtrequested] { + acceptableRequestedMIMEs = true + break + } + } + if !acceptableRequestedMIMEs { + requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes + } return &dockerImageSource{ ref: ref, requestedManifestMIMETypes: requestedManifestMIMETypes, diff --git a/image/docker_schema2.go b/image/docker_schema2.go index a40d53b241..92071de021 100644 --- a/image/docker_schema2.go +++ b/image/docker_schema2.go @@ -12,6 +12,7 @@ import ( "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -166,6 +167,8 @@ func (m *manifestSchema2) UpdatedImage(options types.ManifestUpdateOptions) (typ case "": // No conversion, OK case manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType: return copy.convertToManifestSchema1(options.InformationOnly.Destination) + case imgspecv1.MediaTypeImageManifest: + return copy.convertToManifestOCI1() default: return nil, errors.Errorf("Conversion of image manifest from %s to %s is not implemented", manifest.DockerV2Schema2MediaType, options.ManifestMIMEType) } @@ -173,6 +176,31 @@ func (m *manifestSchema2) UpdatedImage(options types.ManifestUpdateOptions) (typ return memoryImageFromManifest(©), nil } +func (m *manifestSchema2) convertToManifestOCI1() (types.Image, error) { + // Create a copy of the descriptor. + config := m.ConfigDescriptor + + // The only difference between OCI and DockerSchema2 is the mediatypes. The + // media type of the manifest is handled by manifestSchema2FromComponents. + config.MediaType = imgspecv1.MediaTypeImageConfig + + layers := make([]descriptor, len(m.LayersDescriptors)) + for idx := range layers { + layers[idx] = m.LayersDescriptors[idx] + if m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable + } else { + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip + } + } + + // Rather than copying the ConfigBlob now, we just pass m.src to the + // translated manifest, since the only difference is the mediatype of + // descriptors there is no change to any blob stored in m.src. + m1 := manifestOCI1FromComponents(config, m.src, nil, layers) + return memoryImageFromManifest(m1), nil +} + // Based on docker/distribution/manifest/schema1/config_builder.go func (m *manifestSchema2) convertToManifestSchema1(dest types.ImageDestination) (types.Image, error) { configBytes, err := m.ConfigBlob() diff --git a/image/docker_schema2_test.go b/image/docker_schema2_test.go index 6f1f0fbd25..52ef308929 100644 --- a/image/docker_schema2_test.go +++ b/image/docker_schema2_test.go @@ -15,6 +15,7 @@ import ( "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -440,6 +441,28 @@ func TestManifestSchema2UpdatedImage(t *testing.T) { assert.Equal(t, *typedM2, *typedOriginal) } +func TestConvertToManifestOCI(t *testing.T) { + originalSrc := newSchema2ImageSource(t, "httpd-copy:latest") + original := manifestSchema2FromFixture(t, originalSrc, "schema2.json") + res, err := original.UpdatedImage(types.ManifestUpdateOptions{ + ManifestMIMEType: imgspecv1.MediaTypeImageManifest, + }) + require.NoError(t, err) + + convertedJSON, mt, err := res.Manifest() + require.NoError(t, err) + assert.Equal(t, imgspecv1.MediaTypeImageManifest, mt) + + byHandJSON, err := ioutil.ReadFile("fixtures/schema2-to-oci1.json") + require.NoError(t, err) + var converted, byHand map[string]interface{} + err = json.Unmarshal(byHandJSON, &byHand) + require.NoError(t, err) + err = json.Unmarshal(convertedJSON, &converted) + require.NoError(t, err) + assert.Equal(t, byHand, converted) +} + func TestConvertToManifestSchema1(t *testing.T) { originalSrc := newSchema2ImageSource(t, "httpd-copy:latest") original := manifestSchema2FromFixture(t, originalSrc, "schema2.json") diff --git a/image/fixtures/oci1.json b/image/fixtures/oci1.json index e8c2bc791c..d26561d820 100644 --- a/image/fixtures/oci1.json +++ b/image/fixtures/oci1.json @@ -1,6 +1,5 @@ { "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.manifest.v1+json", "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "size": 5940, diff --git a/image/fixtures/schema2-to-oci1.json b/image/fixtures/schema2-to-oci1.json new file mode 100644 index 0000000000..20cdb6e30a --- /dev/null +++ b/image/fixtures/schema2-to-oci1.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 5940, + "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f" + }, + "layers": [{ + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 51354364, + "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb" + }, { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 150, + "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c" + }, { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 11739507, + "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9" + }, { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 8841833, + "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909" + }, { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 291, + "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa" + }] +} diff --git a/image/oci.go b/image/oci.go index 969bb61a02..9c10c5f0cd 100644 --- a/image/oci.go +++ b/image/oci.go @@ -15,7 +15,6 @@ type manifestOCI1 struct { src types.ImageSource // May be nil if configBlob is not nil configBlob []byte // If set, corresponds to contents of ConfigDescriptor. SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` ConfigDescriptor descriptor `json:"config"` LayersDescriptors []descriptor `json:"layers"` } @@ -29,12 +28,11 @@ func manifestOCI1FromManifest(src types.ImageSource, manifest []byte) (genericMa } // manifestOCI1FromComponents builds a new manifestOCI1 from the supplied data: -func manifestOCI1FromComponents(config descriptor, configBlob []byte, layers []descriptor) genericManifest { +func manifestOCI1FromComponents(config descriptor, src types.ImageSource, configBlob []byte, layers []descriptor) genericManifest { return &manifestOCI1{ - src: nil, + src: src, configBlob: configBlob, SchemaVersion: 2, - MediaType: imgspecv1.MediaTypeImageManifest, ConfigDescriptor: config, LayersDescriptors: layers, } @@ -45,7 +43,7 @@ func (m *manifestOCI1) serialize() ([]byte, error) { } func (m *manifestOCI1) manifestMIMEType() string { - return m.MediaType + return imgspecv1.MediaTypeImageManifest } // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. diff --git a/image/oci_test.go b/image/oci_test.go index 2f89643c5d..ac74fbec13 100644 --- a/image/oci_test.go +++ b/image/oci_test.go @@ -34,29 +34,29 @@ func manifestOCI1FromComponentsLikeFixture(configBlob []byte) genericManifest { MediaType: imgspecv1.MediaTypeImageConfig, Size: 5940, Digest: "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f", - }, configBlob, []descriptor{ + }, nil, configBlob, []descriptor{ { - MediaType: imgspecv1.MediaTypeImageLayer, + MediaType: imgspecv1.MediaTypeImageLayerGzip, Digest: "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb", Size: 51354364, }, { - MediaType: imgspecv1.MediaTypeImageLayer, + MediaType: imgspecv1.MediaTypeImageLayerGzip, Digest: "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c", Size: 150, }, { - MediaType: imgspecv1.MediaTypeImageLayer, + MediaType: imgspecv1.MediaTypeImageLayerGzip, Digest: "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9", Size: 11739507, }, { - MediaType: imgspecv1.MediaTypeImageLayer, + MediaType: imgspecv1.MediaTypeImageLayerGzip, Digest: "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909", Size: 8841833, }, { - MediaType: imgspecv1.MediaTypeImageLayer, + MediaType: imgspecv1.MediaTypeImageLayerGzip, Digest: "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa", Size: 291, }, diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 4f4a12c003..06ae40ffa2 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -37,7 +37,6 @@ func (d *ociImageDestination) Close() { func (d *ociImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, - manifest.DockerV2Schema2MediaType, } } @@ -134,60 +133,16 @@ func (d *ociImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, return info, nil } -func createManifest(m []byte) ([]byte, string, error) { - om := imgspecv1.Manifest{} - mt := manifest.GuessMIMEType(m) - switch mt { - case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType: - // There a simple reason about not yet implementing this. - // OCI image-spec assure about backward compatibility with docker v2s2 but not v2s1 - // generating a v2s2 is a migration docker does when upgrading to 1.10.3 - // and I don't think we should bother about this now (I don't want to have migration code here in skopeo) - return nil, "", errors.New("can't create an OCI manifest from Docker V2 schema 1 manifest") - case manifest.DockerV2Schema2MediaType: - if err := json.Unmarshal(m, &om); err != nil { - return nil, "", err - } - om.MediaType = imgspecv1.MediaTypeImageManifest - for i, l := range om.Layers { - if l.MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { - om.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable - } else { - om.Layers[i].MediaType = imgspecv1.MediaTypeImageLayer - } - } - om.Config.MediaType = imgspecv1.MediaTypeImageConfig - b, err := json.Marshal(om) - if err != nil { - return nil, "", err - } - return b, om.MediaType, nil - case manifest.DockerV2ListMediaType: - return nil, "", errors.New("can't create an OCI manifest from Docker V2 schema 2 manifest list") - case imgspecv1.MediaTypeImageManifestList: - return nil, "", errors.New("can't create an OCI manifest from OCI manifest list") - case imgspecv1.MediaTypeImageManifest: - return m, mt, nil - } - return nil, "", errors.Errorf("unrecognized manifest media type %q", mt) -} - func (d *ociImageDestination) PutManifest(m []byte) error { - // TODO(mitr, runcom): this breaks signatures entirely since at this point we're creating a new manifest - // and signatures don't apply anymore. Will fix. - ociMan, mt, err := createManifest(m) - if err != nil { - return err - } - digest, err := manifest.Digest(ociMan) + digest, err := manifest.Digest(m) if err != nil { return err } desc := imgspecv1.Descriptor{} - desc.Digest = digest.String() + desc.Digest = digest // TODO(runcom): beaware and add support for OCI manifest list - desc.MediaType = mt - desc.Size = int64(len(ociMan)) + desc.MediaType = imgspecv1.MediaTypeImageManifest + desc.Size = int64(len(m)) data, err := json.Marshal(desc) if err != nil { return err @@ -197,7 +152,10 @@ func (d *ociImageDestination) PutManifest(m []byte) error { if err != nil { return err } - if err := ioutil.WriteFile(blobPath, ociMan, 0644); err != nil { + if err := ensureParentDirectoryExists(blobPath); err != nil { + return err + } + if err := ioutil.WriteFile(blobPath, m, 0644); err != nil { return err } // TODO(runcom): ugly here? diff --git a/oci/layout/oci_dest_test.go b/oci/layout/oci_dest_test.go index 6b039e41f8..9767f94f3a 100644 --- a/oci/layout/oci_dest_test.go +++ b/oci/layout/oci_dest_test.go @@ -59,36 +59,3 @@ func TestPutBlobDigestFailure(t *testing.T) { require.Error(t, err) require.True(t, os.IsNotExist(err)) } - -func TestPutManifestUnrecognizedMediaType(t *testing.T) { - ref, tmpDir := refToTempOCI(t) - defer os.RemoveAll(tmpDir) - dirRef, ok := ref.(ociReference) - require.True(t, ok) - - ociDest, err := dirRef.NewImageDestination(nil) - require.NoError(t, err) - - m := `{"name":"puerapuliae/busybox","tag":"latest","architecture":"amd64","fsLayers":[{"blobSum":"sha256:04f18047a28f8dea4a3b3872a2ad345cbb6f0eae28d99a60d3df844d6eaae571"},{"blobSum":"sha256:04f18047a28f8dea4a3b3872a2ad345cbb6f0eae28d99a60d3df844d6eaae571"}],"history":[{"v1Compatibility":"{\"id\":\"b46e47334e74d687019107dbec32559dd598db58fe90d2a0c5473bda8b59829d\",\"comment\":\"Imported from -\",\"created\":\"2015-07-03T07:56:02.57018886Z\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":9356886}\n"},{"v1Compatibility":"{\"id\":\"b46e47334e74d687019107dbec32559dd598db58fe90d2a0c5473bda8b59829d\",\"comment\":\"Imported from -\",\"created\":\"2015-07-03T07:56:02.57018886Z\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":9356886}\n"}],"signatures":[{"header":{"jwk":{"crv":"P-256","kid":"SVJ4:Q6G3:SXTN:H6LT:7PXH:DHUZ:SGTB:5TMV:YPIV:UPHY:MRHO:PN6V","kty":"EC","x":"qrSsA2UAKEFlDhLk12zoWpnHgYcTNfEOWGZU46pzhfk","y":"RtD_vGFtagPlheiunLvZL02LOssnu7DqShuBwc6Ml44"},"alg":"ES256"},"signature":"YzfU_rKQLWqG74uilltTiV3O92lfEjaG5wJkVt_dCtjH_C5AeghfQttnbtceJOyiaU7xP2yEnjdultutsxkQKQ","protected":"eyJmb3JtYXRMZW5ndGgiOjI4NDgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNi0wOS0xMFQwODoyMDowOFoifQ"}]}` - - err = ociDest.PutManifest([]byte(m)) - require.Error(t, err) - assert.Equal(t, `unrecognized manifest media type ""`, err.Error()) -} - -// regression test for projectatomic/skopeo#198 -func TestPutManifestDockerV2Schema1Signed(t *testing.T) { - ref, tmpDir := refToTempOCI(t) - defer os.RemoveAll(tmpDir) - dirRef, ok := ref.(ociReference) - require.True(t, ok) - - ociDest, err := dirRef.NewImageDestination(nil) - require.NoError(t, err) - - m := `{"name":"puerapuliae/busybox","tag":"latest","architecture":"amd64","fsLayers":[{"blobSum":"sha256:04f18047a28f8dea4a3b3872a2ad345cbb6f0eae28d99a60d3df844d6eaae571"},{"blobSum":"sha256:04f18047a28f8dea4a3b3872a2ad345cbb6f0eae28d99a60d3df844d6eaae571"}],"history":[{"v1Compatibility":"{\"id\":\"b46e47334e74d687019107dbec32559dd598db58fe90d2a0c5473bda8b59829d\",\"comment\":\"Imported from -\",\"created\":\"2015-07-03T07:56:02.57018886Z\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":9356886}\n"},{"v1Compatibility":"{\"id\":\"b46e47334e74d687019107dbec32559dd598db58fe90d2a0c5473bda8b59829d\",\"comment\":\"Imported from -\",\"created\":\"2015-07-03T07:56:02.57018886Z\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":9356886}\n"}],"schemaVersion":1,"signatures":[{"header":{"jwk":{"crv":"P-256","kid":"SVJ4:Q6G3:SXTN:H6LT:7PXH:DHUZ:SGTB:5TMV:YPIV:UPHY:MRHO:PN6V","kty":"EC","x":"qrSsA2UAKEFlDhLk12zoWpnHgYcTNfEOWGZU46pzhfk","y":"RtD_vGFtagPlheiunLvZL02LOssnu7DqShuBwc6Ml44"},"alg":"ES256"},"signature":"YzfU_rKQLWqG74uilltTiV3O92lfEjaG5wJkVt_dCtjH_C5AeghfQttnbtceJOyiaU7xP2yEnjdultutsxkQKQ","protected":"eyJmb3JtYXRMZW5ndGgiOjI4NDgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNi0wOS0xMFQwODoyMDowOFoifQ"}]}` - - err = ociDest.PutManifest([]byte(m)) - require.Error(t, err) - assert.Equal(t, `can't create an OCI manifest from Docker V2 schema 1 manifest`, err.Error()) -} diff --git a/openshift/openshift-copies.go b/openshift/openshift-copies.go index 297d462869..01fe71a243 100644 --- a/openshift/openshift-copies.go +++ b/openshift/openshift-copies.go @@ -13,14 +13,14 @@ import ( "path" "path/filepath" "reflect" + "strings" "time" "github.com/ghodss/yaml" "github.com/imdario/mergo" "github.com/pkg/errors" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/kubernetes/pkg/util/homedir" + "golang.org/x/net/http2" + "k8s.io/client-go/util/homedir" ) // restTLSClientConfig is a modified copy of k8s.io/kubernets/pkg/client/restclient.TLSClientConfig. @@ -450,6 +450,55 @@ func (config *directClientConfig) getCluster() clientcmdCluster { return mergedClusterInfo } +// aggregateErr is a modified copy of k8s.io/apimachinery/pkg/util/errors.aggregate. +// This helper implements the error and Errors interfaces. Keeping it private +// prevents people from making an aggregate of 0 errors, which is not +// an error, but does satisfy the error interface. +type aggregateErr []error + +// newAggregate is a modified copy of k8s.io/apimachinery/pkg/util/errors.NewAggregate. +// NewAggregate converts a slice of errors into an Aggregate interface, which +// is itself an implementation of the error interface. If the slice is empty, +// this returns nil. +// It will check if any of the element of input error list is nil, to avoid +// nil pointer panic when call Error(). +func newAggregate(errlist []error) error { + if len(errlist) == 0 { + return nil + } + // In case of input error list contains nil + var errs []error + for _, e := range errlist { + if e != nil { + errs = append(errs, e) + } + } + if len(errs) == 0 { + return nil + } + return aggregateErr(errs) +} + +// Error is a modified copy of k8s.io/apimachinery/pkg/util/errors.aggregate.Error. +// Error is part of the error interface. +func (agg aggregateErr) Error() string { + if len(agg) == 0 { + // This should never happen, really. + return "" + } + if len(agg) == 1 { + return agg[0].Error() + } + result := fmt.Sprintf("[%s", agg[0].Error()) + for i := 1; i < len(agg); i++ { + result += fmt.Sprintf(", %s", agg[i].Error()) + } + result += "]" + return result +} + +// REMOVED: aggregateErr.Errors + // errConfigurationInvalid is a modified? copy of k8s.io/kubernetes/pkg/client/unversioned/clientcmd.errConfigurationInvalid. // errConfigurationInvalid is a set of errors indicating the configuration is invalid. type errConfigurationInvalid []error @@ -470,7 +519,7 @@ func newErrConfigurationInvalid(errs []error) error { // Error implements the error interface func (e errConfigurationInvalid) Error() string { - return fmt.Sprintf("invalid configuration: %v", utilerrors.NewAggregate(e).Error()) + return fmt.Sprintf("invalid configuration: %v", newAggregate(e).Error()) } // clientConfigLoadingRules is a modified copy of k8s.io/kubernetes/pkg/client/unversioned/clientcmd.ClientConfigLoadingRules @@ -550,7 +599,7 @@ func (rules *clientConfigLoadingRules) Load() (*clientcmdConfig, error) { errlist = append(errlist, err) } - return config, utilerrors.NewAggregate(errlist) + return config, newAggregate(errlist) } // loadFromFile is a modified copy of k8s.io/kubernetes/pkg/client/unversioned/clientcmd.LoadFromFile @@ -799,6 +848,52 @@ func transportNew(config *restConfig) (http.RoundTripper, error) { return rt, nil } +// newProxierWithNoProxyCIDR is a modified copy of k8s.io/apimachinery/pkg/util/net.NewProxierWithNoProxyCIDR. +// NewProxierWithNoProxyCIDR constructs a Proxier function that respects CIDRs in NO_PROXY and delegates if +// no matching CIDRs are found +func newProxierWithNoProxyCIDR(delegate func(req *http.Request) (*url.URL, error)) func(req *http.Request) (*url.URL, error) { + // we wrap the default method, so we only need to perform our check if the NO_PROXY envvar has a CIDR in it + noProxyEnv := os.Getenv("NO_PROXY") + noProxyRules := strings.Split(noProxyEnv, ",") + + cidrs := []*net.IPNet{} + for _, noProxyRule := range noProxyRules { + _, cidr, _ := net.ParseCIDR(noProxyRule) + if cidr != nil { + cidrs = append(cidrs, cidr) + } + } + + if len(cidrs) == 0 { + return delegate + } + + return func(req *http.Request) (*url.URL, error) { + host := req.URL.Host + // for some urls, the Host is already the host, not the host:port + if net.ParseIP(host) == nil { + var err error + host, _, err = net.SplitHostPort(req.URL.Host) + if err != nil { + return delegate(req) + } + } + + ip := net.ParseIP(host) + if ip == nil { + return delegate(req) + } + + for _, cidr := range cidrs { + if cidr.Contains(ip) { + return nil, nil + } + } + + return delegate(req) + } +} + // tlsCacheGet is a modified copy of k8s.io/kubernetes/pkg/client/transport.tlsTransportCache.get. func tlsCacheGet(config *restConfig) (http.RoundTripper, error) { // REMOVED: any actual caching @@ -813,15 +908,23 @@ func tlsCacheGet(config *restConfig) (http.RoundTripper, error) { return http.DefaultTransport, nil } - return utilnet.SetTransportDefaults(&http.Transport{ // FIXME?? - Proxy: http.ProxyFromEnvironment, + // REMOVED: Call to k8s.io/apimachinery/pkg/util/net.SetTransportDefaults; instead of the generic machinery and conditionals, hard-coded the result here. + t := &http.Transport{ + // http.ProxyFromEnvironment doesn't respect CIDRs and that makes it impossible to exclude things like pod and service IPs from proxy settings + // ProxierWithNoProxyCIDR allows CIDR rules in NO_PROXY + Proxy: newProxierWithNoProxyCIDR(http.ProxyFromEnvironment), TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: tlsConfig, Dial: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).Dial, - }), nil + } + // Allow clients to disable http2 if needed. + if s := os.Getenv("DISABLE_HTTP2"); len(s) == 0 { + _ = http2.ConfigureTransport(t) + } + return t, nil } // tlsConfigFor is a modified copy of k8s.io/kubernetes/pkg/client/transport.TLSConfigFor. diff --git a/storage/storage_test.go b/storage/storage_test.go index 78dafc15f6..6d17728d24 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -83,8 +83,8 @@ func newStore(t *testing.T) storage.Store { GraphRoot: root, GraphDriverName: "vfs", GraphDriverOptions: []string{}, - UidMap: uidmap, - GidMap: gidmap, + UIDMap: uidmap, + GIDMap: gidmap, }) if err != nil { t.Fatal(err)