diff --git a/Dockerfile b/Dockerfile index 81d52810e0..1e7dcfc2e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-go-md swig \ redhat-rpm-config \ openssl-devel \ - patch + patch \ + jq # Install three versions of the registry. The first is an older version that # only supports schema1 manifests. The second is a newer version that supports @@ -26,16 +27,7 @@ RUN set -x \ && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1") \ && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ go build -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \ - && rm -rf "$GOPATH" \ - && export DRV1="$(mktemp -d)" \ - && git clone https://github.com/docker/docker-registry.git "$DRV1" \ - && sed -i.bak s/setuptools==5.8//g "$DRV1/requirements/main.txt" \ - && sed -i.bak s/setuptools==5.8//g "$DRV1/depends/docker-registry-core/requirements/main.txt" \ - && pip install "$DRV1/depends/docker-registry-core" \ - && pip install file://"$DRV1#egg=docker-registry[bugsnag,newrelic,cors]" \ - && patch $(python -c 'import boto; import os; print os.path.dirname(boto.__file__)')/connection.py \ - < "$DRV1/contrib/boto_header_patch.diff" \ - && dnf -y update && dnf install -y m2crypto + && rm -rf "$GOPATH" RUN set -x \ && yum install -y which git tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \ @@ -46,10 +38,18 @@ RUN set -x \ && cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \ && mkdir /registry + ENV GOPATH /usr/share/gocode:/go ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH RUN go get github.com/golang/lint/golint WORKDIR /go/src/github.com/projectatomic/skopeo COPY . /go/src/github.com/projectatomic/skopeo +# Get useful and necessary Hub images so we can "docker load" locally instead of pulling +RUN ./hack/download-frozen-image-v2.sh /docker-frozen-images \ + buildpack-deps:jessie@sha256:25785f89240fbcdd8a74bdaf30dd5599a9523882c6dfc567f2e9ef7cf6f79db6 \ + busybox:latest@sha256:e4f93f6ed15a0cdd342f5aae387886fba0ab98af0a102da6276eaf24d6e6ade0 \ + debian:jessie@sha256:f968f10b4b523737e253a97eac59b0d1420b5c19b69928d35801a6373ffe330e \ + hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7 + #ENTRYPOINT ["hack/dind"] diff --git a/hack/download-frozen-image-v2.sh b/hack/download-frozen-image-v2.sh new file mode 100755 index 0000000000..111e3fa2ba --- /dev/null +++ b/hack/download-frozen-image-v2.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -e + +# hello-world latest ef872312fe1b 3 months ago 910 B +# hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B + +# debian latest f6fab3b798be 10 weeks ago 85.1 MB +# debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB + +if ! command -v curl &> /dev/null; then + echo >&2 'error: "curl" not found!' + exit 1 +fi + +usage() { + echo "usage: $0 dir image[:tag][@digest] ..." + echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7" + [ -z "$1" ] || exit "$1" +} + +dir="$1" # dir for building tar in +shift || usage 1 >&2 + +[ $# -gt 0 -a "$dir" ] || usage 2 >&2 +mkdir -p "$dir" + +# hacky workarounds for Bash 3 support (no associative arrays) +images=() +rm -f "$dir"/tags-*.tmp +# repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' + +while [ $# -gt 0 ]; do + imageTag="$1" + shift + image="${imageTag%%[:@]*}" + imageTag="${imageTag#*:}" + digest="${imageTag##*@}" + tag="${imageTag%%@*}" + + # add prefix library if passed official image + if [[ "$image" != *"/"* ]]; then + image="library/$image" + fi + + imageFile="${image//\//_}" # "/" can't be in filenames :) + + token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" + + manifestJson="$(curl -sSL -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/manifests/$digest")" + if [ "${manifestJson:0:1}" != '{' ]; then + echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" + echo >&2 " $manifestJson" + exit 1 + fi + + layersFs=$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum') + + IFS=$'\n' + # bash v4 on Windows CI requires CRLF separator + if [ "$(go env GOHOSTOS)" = 'windows' ]; then + major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1) + if [ "$major" -ge 4 ]; then + IFS=$'\r\n' + fi + fi + layers=( ${layersFs} ) + unset IFS + + history=$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]') + imageId=$(echo "$history" | jq --raw-output .[0] | jq --raw-output .id) + + if [ -s "$dir/tags-$imageFile.tmp" ]; then + echo -n ', ' >> "$dir/tags-$imageFile.tmp" + else + images=( "${images[@]}" "$image" ) + fi + echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" + + echo "Downloading '${image}:${tag}@${digest}' (${#layers[@]} layers)..." + for i in "${!layers[@]}"; do + imageJson=$(echo "$history" | jq --raw-output .[${i}]) + imageId=$(echo "$imageJson" | jq --raw-output .id) + imageLayer=${layers[$i]} + + mkdir -p "$dir/$imageId" + echo '1.0' > "$dir/$imageId/VERSION" + + echo "$imageJson" > "$dir/$imageId/json" + + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$imageId/layer.tar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${imageId:0:12}" + continue + fi + token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" + curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C - + done + echo +done + +echo -n '{' > "$dir/repositories" +firstImage=1 +for image in "${images[@]}"; do + imageFile="${image//\//_}" # "/" can't be in filenames :) + image="${image#library\/}" + + [ "$firstImage" ] || echo -n ',' >> "$dir/repositories" + firstImage= + echo -n $'\n\t' >> "$dir/repositories" + echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories" +done +echo -n $'\n}\n' >> "$dir/repositories" + +rm -f "$dir"/tags-*.tmp + +echo "Download of images into '$dir' complete." +echo "Use something like the following to load the result into a Docker daemon:" +echo " tar -cC '$dir' . | docker load" diff --git a/integration/check_test.go b/integration/check_test.go index e3869fbad8..92af23f09a 100644 --- a/integration/check_test.go +++ b/integration/check_test.go @@ -26,10 +26,8 @@ func init() { } type SkopeoSuite struct { - regV1 *testRegistryV1 regV2 *testRegistryV2 regV2Shema1 *testRegistryV2 - regV1WithAuth *testRegistryV1 // does v1 support auth? regV2WithAuth *testRegistryV2 } @@ -45,15 +43,12 @@ func (s *SkopeoSuite) SetUpTest(c *check.C) { _, err := exec.LookPath(skopeoBinary) c.Assert(err, check.IsNil) - s.regV1 = setupRegistryV1At(c, privateRegistryURL0, false) // TODO:(runcom) s.regV2 = setupRegistryV2At(c, privateRegistryURL1, false, false) s.regV2Shema1 = setupRegistryV2At(c, privateRegistryURL2, false, true) - s.regV1WithAuth = setupRegistryV1At(c, privateRegistryURL3, true) // not used s.regV2WithAuth = setupRegistryV2At(c, privateRegistryURL4, true, false) } func (s *SkopeoSuite) TearDownTest(c *check.C) { - // not checking V1 registries now... if s.regV2 != nil { s.regV2.Close() } diff --git a/integration/registry.go b/integration/registry.go index 0136e33615..30aec31e89 100644 --- a/integration/registry.go +++ b/integration/registry.go @@ -13,23 +13,10 @@ import ( ) const ( - binaryV1 = "docker-registry" binaryV2 = "registry-v2" binaryV2Schema1 = "registry-v2-schema1" ) -type testRegistryV1 struct { - cmd *exec.Cmd - url string - dir string -} - -func setupRegistryV1At(c *check.C, url string, auth bool) *testRegistryV1 { - return &testRegistryV1{ - url: url, - } -} - type testRegistryV2 struct { cmd *exec.Cmd url string