moby · crazy-max · Jun 7, 2022 · Jun 14, 2022
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -27,7 +27,10 @@ env:
   CACHE_GHA_SCOPE_IT: "integration-tests"
   CACHE_GHA_SCOPE_BINARIES: "binaries"
   CACHE_GHA_SCOPE_CROSS: "cross"
+  GO_VERSION: "1.18"  # same as one in Dockerfile
+  GOTESTSUM_VERSION: "v1.8.1"  # same as one in Dockerfile
   BUILDX_VERSION: "v0.8.2"  # leave empty to use the one available on GitHub virtual environment
+  TESTSTAT_VERSION: "v0.1.3"
 
 jobs:
   base:
@@ -112,23 +115,28 @@ jobs:
       -
         name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; skipit=${{ matrix.skip-integration-tests }} ; worker=${{ matrix.worker }}
         run: |
-          export TESTFLAGS="-v --parallel=6 --timeout=20m"
+          export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
+          export TESTFLAGS="-v --parallel=6 --timeout=30m"
           if [ -n "${{ matrix.worker }}" ]; then
             export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$"
           fi
           ./hack/test ${{ matrix.typ }}
-          mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt
         env:
-          TEST_COVERAGE: 1
           TESTPKGS: ${{ matrix.pkg }}
           SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
           CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}
       -
-        name: Upload coverage file
-        uses: actions/upload-artifact@v2
+        name: Send to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ./bin/testreports
+      -
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
         with:
-          name: coverage
-          path: ./coverage
+          name: test-reports
+          path: ./bin/testreports
 
   test-s3:
     runs-on: ubuntu-latest
@@ -170,7 +178,7 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: ${{ env.GO_VERSION }}
       -
         name: Cache Go modules
         uses: actions/cache@v2
@@ -179,6 +187,10 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+      -
+        name: Install gotestsum
+        run: |
+          go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}
       -
         name: Go mod
         run: |
@@ -189,45 +201,82 @@ jobs:
           TMPDIR: ${{ runner.temp }}
           SKIP_INTEGRATION_TESTS: 1
         run: |
-          mkdir -p ./coverage
-          go test -coverprofile=./coverage/coverage-${{ github.job }}-${{ matrix.os }}.txt -covermode=atomic ./...
+          TEST_REPORT_SUFFIX=-${{ github.job }}-${{ matrix.os }}
+          mkdir -p ./bin/testreports
+          gotestsum \
+            --format=standard-verbose \
+            --jsonfile="./bin/testreports/go-test-report${TEST_REPORT_SUFFIX}.json" \
+            --junitfile="./bin/testreports/junit-report${TEST_REPORT_SUFFIX}.xml" \
+            --packages="./..." \
+            -- \
+              "-coverprofile" "./bin/testreports/coverage-report${TEST_REPORT_SUFFIX}.txt" \
+              "-covermode" "atomic"
         shell: bash
       -
-        name: Upload coverage file
-        uses: actions/upload-artifact@v2
+        name: Send to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: ./bin/testreports
+      -
+        name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v3
         with:
-          name: coverage
-          path: ./coverage
+          name: test-reports
+          path: ./bin/testreports
 
-  upload-coverage:
+  test-report:
     runs-on: ubuntu-latest
-    needs: [test, test-os]
+    if: always()
+    needs:
+      - test
+      - test-os
+    strategy:
+      fail-fast: false
+      matrix:
+        filter:
+          - unit
+          - integration-containerd
+          - integration-containerd-14
+          - integration-containerd-15
+          - integration-containerd-rootless
+          - integration-containerd-snapshotter-stargz
+          - integration-oci
+          - integration-oci-rootless
+          - integration-oci-snapshotter-stargz
     steps:
       -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Download coverage files
-        uses: actions/download-artifact@v2
+        name: Set up Go
+        uses: actions/setup-go@v3
         with:
-          name: coverage
-          path: ./coverage
+          go-version: ${{ env.GO_VERSION }}
       -
-        name: List coverage files
-        uses: actions/github-script@v3
-        id: files
+        name: Download artifacts
+        uses: actions/download-artifact@v3
         with:
-          result-encoding: string
-          script: |
-            return require('fs').readdirSync('./coverage', {withFileTypes: true})
-              .filter(item => !item.isDirectory())
-              .map(item => `./coverage/${item.name}`)
-              .join(',');
+          name: test-reports
+          path: /tmp/testreports
       -
-        name: Send to Codecov
-        uses: codecov/codecov-action@v2
+        name: Install teststat
+        run: |
+          go install github.com/vearutop/teststat@${{ env.TESTSTAT_VERSION }}
+      -
+        name: Filter test reports
+        run: |
+          filter=(! -name "*${{ matrix.filter}}.*")
+          if [ "${{ matrix.filter}}" = "unit" ]; then
+            filter=(! -name "*integrationgateway-.*" -a ! -name "*test-os-*.*")
+          fi
+          find /tmp/testreports -type f ${filter[@]} -delete
+      -
+        name: Run test-summary
+        uses: test-summary/action@v1
         with:
-          files: ${{ steps.files.outputs.result }}
+          paths: /tmp/testreports/**/*.xml
+      -
+        name: Run teststat
+        run: |
+          teststat -markdown $(find /tmp/testreports -type f -name "*.json" -print0 | xargs -0) >> $GITHUB_STEP_SUMMARY
 
   cross:
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,4 @@
 bin
-coverage
 release-out
 .certs
 .tmp
diff --git a/Dockerfile b/Dockerfile
@@ -13,6 +13,7 @@ ARG ROOTLESSKIT_VERSION=v0.14.6
 ARG CNI_VERSION=v1.1.0
 ARG STARGZ_SNAPSHOTTER_VERSION=v0.11.4
 ARG NERDCTL_VERSION=v0.17.1
+ARG GOTESTSUM_VERSION=v1.8.1
 
 # ALPINE_VERSION sets version for the base layers
 ARG ALPINE_VERSION=3.15
@@ -184,6 +185,12 @@ RUN --mount=target=/root/.cache,type=cache \
   xx-verify --static /out/containerd-stargz-grpc && \
   xx-verify --static /out/ctr-remote
 
+FROM gobuild-base AS gotestsum
+ARG GOTESTSUM_VERSION
+RUN --mount=target=/root/.cache,type=cache \
+  GOBIN=/out/ go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" && \
+  /out/gotestsum --version
+
 # Copy together all binaries needed for oci worker mode
 FROM buildkit-export AS buildkit-buildkitd.oci_only
 COPY --link --from=buildkitd.oci_only /usr/bin/buildkitd.oci_only /usr/bin/
@@ -241,6 +248,7 @@ RUN apk add --no-cache shadow shadow-uidmap sudo vim iptables fuse curl \
 ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.4=/opt/containerd-alt-14/bin,containerd-1.5=/opt/containerd-alt-15/bin"
 ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV CGO_ENABLED=0
+COPY --link --from=gotestsum /out/gotestsum /usr/bin/
 COPY --link --from=stargz-snapshotter /out/* /usr/bin/
 COPY --link --from=rootlesskit /rootlesskit /usr/bin/
 COPY --link --from=containerd-alt-14 /out/containerd* /opt/containerd-alt-14/bin/

diff --git a/hack/test b/hack/test
@@ -8,7 +8,7 @@ set -eu -o pipefail
 : ${TEST_DOCKERFILE=}
 : ${TEST_DOCKERD=}
 : ${TEST_DOCKERD_BINARY=$(which dockerd)}
-: ${TEST_COVERAGE=}
+: ${TEST_REPORT_SUFFIX=}
 : ${TEST_KEEP_CACHE=}
 : ${DOCKERFILE_RELEASES=}
 : ${BUILDKIT_WORKER_RANDOM=}
@@ -50,14 +50,11 @@ done
 iid="buildkit-tests"
 iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
 
-coverageVol=""
-coverageFlags=""
-if [ "$TEST_COVERAGE" = "1" ]; then
-  covdir="$(pwd)/coverage"
-  mkdir -p "$covdir"
-  coverageVol="-v $covdir:/coverage"
-  coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
-fi
+testReportsDir="$(pwd)/bin/testreports"
+mkdir -p "$testReportsDir"
+testReportsVol="-v $testReportsDir:/testreports"
+gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
+gotestArgs="-coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
 
 buildxCmd build $cacheFromFlags \
   --target "integration-tests" \
@@ -70,7 +67,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
 fi
 
 if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
+  cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
   fi
@@ -110,7 +107,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
 
     if [ -s $tarout ]; then
       if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
-        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
+        cid=$(docker create -v /tmp $testReportsVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         docker cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then
           docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd