conformance tests: more parallelization

contrib/ci/logcollector.sh

@@ -33,8 +33,8 @@ case $1 in
     df) showrun df -lhTx tmpfs ;;
     journal) showrun journalctl -b ;;
     podman) showrun podman system info ;;
-    buildah_version) showrun "$GOSRC/bin/buildah" version ;;
-    buildah_info) showrun "$GOSRC/bin/buildah" info ;;
+    buildah_version) make bin/buildah && showrun "$GOSRC/bin/buildah" version ;;
+    buildah_info) make bin/buildah && showrun "$GOSRC/bin/buildah" info ;;
     golang) showrun go version ;;
     packages)
         PKG_NAMES=(\

contrib/ci/runner.sh

@@ -98,15 +98,13 @@ function run_conformance() {
 }
 
 function run_integration() {
-    make all
     $SUDO make test-integration
 }
 
 function run_in_podman() {
     export IN_PODMAN=true
     export BUILDAH_ISOLATION=chroot
     export STORAGE_DRIVER=vfs
-    make all
     $SUDO make test-integration
 }
 

tests/commit.bats

@@ -210,14 +210,14 @@ load helpers
 
 @test "commit should fail with nonexistent authfile" {
   _prefetch alpine
-  run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah 125 commit --authfile /tmp/nonexistent $WITH_POLICY_JSON $cid alpine-image
 }
 
 @test "commit-builder-identity" {
 	_prefetch alpine
-	run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+	run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
 	cid=$output
 	run_buildah commit $WITH_POLICY_JSON $cid alpine-image
 
@@ -231,7 +231,7 @@ load helpers
 
 @test "commit-container-id" {
   _prefetch alpine
-  run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
 
   # There is exactly one container. Get its ID.
   run_buildah containers --format '{{.ContainerID}}'

tests/config.bats

@@ -372,7 +372,7 @@ EOF
 
 @test "user" {
   _prefetch alpine
-  run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run $cid grep CapBnd /proc/self/status
   bndoutput=$output

tests/conformance/conformance_test.go

@@ -152,7 +152,6 @@ func TestMain(m *testing.M) {
 }
 
 func TestConformance(t *testing.T) {
-	t.Parallel()
 	dateStamp := fmt.Sprintf("%d", time.Now().UnixNano())
 	for i := range internalTestCases {
 		t.Run(internalTestCases[i].name, func(t *testing.T) {
@@ -196,6 +195,11 @@ func TestConformance(t *testing.T) {
 
 func testConformanceInternal(t *testing.T, dateStamp string, testIndex int, mutate func(*testCase)) {
 	test := internalTestCases[testIndex]
+
+	if !test.dontParallelize {
+		t.Parallel()
+	}
+
 	if mutate != nil {
 		mutate(&test)
 	}
@@ -231,31 +235,26 @@ func testConformanceInternal(t *testing.T, dateStamp string, testIndex int, muta
 
 	// copy either a directory or just a Dockerfile into the temporary directory
 	pipeReader, pipeWriter := io.Pipe()
-	var getErr, putErr error
 	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
+	wg.Go(func() {
 		if test.contextDir != "" {
-			getErr = copier.Get("", testDataDir, copier.GetOptions{}, []string{test.contextDir}, pipeWriter)
+			err := copier.Get("", testDataDir, copier.GetOptions{}, []string{test.contextDir}, pipeWriter)
+			assert.NoErrorf(t, err, "reading context directory %q", filepath.Join("testdata", test.contextDir))
 		} else if test.dockerfile != "" {
-			getErr = copier.Get("", testDataDir, copier.GetOptions{}, []string{test.dockerfile}, pipeWriter)
+			err := copier.Get("", testDataDir, copier.GetOptions{}, []string{test.dockerfile}, pipeWriter)
+			assert.NoErrorf(t, err, "reading dockerfile %q", filepath.Join("testdata", test.dockerfile))
 		}
-		pipeWriter.Close()
-		wg.Done()
-	}()
-	wg.Add(1)
-	go func() {
+	})
+	wg.Go(func() {
 		if test.contextDir != "" || test.dockerfile != "" {
-			putErr = copier.Put("", contextDir, copier.PutOptions{}, pipeReader)
+			err := copier.Put("", contextDir, copier.PutOptions{}, pipeReader)
+			assert.NoErrorf(t, err, "extracting build context at %q", contextDir)
 		} else {
-			putErr = os.Mkdir(contextDir, 0o755)
+			err := os.Mkdir(contextDir, 0o755)
+			assert.NoErrorf(t, err, "creating dummy context directory at %q", contextDir)
 		}
-		pipeReader.Close()
-		wg.Done()
-	}()
+	})
 	wg.Wait()
-	assert.NoErrorf(t, getErr, "error reading build info from %q", filepath.Join("testdata", test.dockerfile))
-	assert.NoErrorf(t, putErr, "error writing build info to %q", contextDir)
 	if t.Failed() {
 		t.FailNow()
 	}
@@ -295,12 +294,12 @@ func testConformanceInternal(t *testing.T, dateStamp string, testIndex int, muta
 	}
 	store, err := storage.GetStore(options)
 	require.NoErrorf(t, err, "error creating buildah storage at %q", rootDir)
-	defer func() {
+	t.Cleanup(func() {
 		if store != nil {
 			_, err := store.Shutdown(true)
 			require.NoError(t, err, "error shutting down storage for buildah")
 		}
-	}()
+	})
 	storageDriver := store.GraphDriverName()
 	storageRoot := store.GraphRoot()
 
@@ -460,13 +459,13 @@ func testConformanceInternalBuild(ctx context.Context, t *testing.T, cwd string,
 	if compareImagebuilder && !test.withoutImagebuilder {
 		imagebuilderRef, imagebuilderLog = buildUsingImagebuilder(t, client, test, imagebuilderImage, contextDir, dockerfileName, line, finalOfSeveral)
 		if imagebuilderRef != nil {
-			defer func() {
+			t.Cleanup(func() {
 				err := client.RemoveImageExtended(imagebuilderImage, docker.RemoveImageOptions{
 					Context: ctx,
 					Force:   true,
 				})
 				assert.Nil(t, err, "error deleting newly-built-by-imagebuilder image %q", imagebuilderImage)
-			}()
+			})
 		}
 		saveReport(ctx, t, imagebuilderRef, filepath.Join(imagebuilderDir, t.Name()), dockerfileContents, imagebuilderLog, dockerVersion)
 		if finalOfSeveral && compareLayers {
@@ -481,10 +480,10 @@ func testConformanceInternalBuild(ctx context.Context, t *testing.T, cwd string,
 	// always build using buildah
 	buildahRef, buildahLog = buildUsingBuildah(ctx, t, store, test, buildahImage, contextDir, dockerfileName, line, finalOfSeveral)
 	if buildahRef != nil {
-		defer func() {
+		t.Cleanup(func() {
 			err := buildahRef.DeleteImage(ctx, nil)
 			assert.Nil(t, err, "error deleting newly-built-by-buildah image %q", buildahImage)
-		}()
+		})
 	}
 	saveReport(ctx, t, buildahRef, filepath.Join(buildahDir, t.Name()), dockerfileContents, buildahLog, nil)
 	if finalOfSeveral && compareLayers {
@@ -1449,6 +1448,7 @@ type (
 
 		fsSkipCompatVolumesTrue []string          // more expected filesystem differences when compatVolumes=true
 		buildArgs               map[string]string // build args to supply, as if --build-arg was used
+		dontParallelize         bool              // uses shared state managed elsewhere
 	}
 )
 
@@ -3794,20 +3794,22 @@ var internalTestCases = []testCase{
 	},
 	{
 		name:              "mount-cache-by-ownership",
+		dontParallelize:   true, // the docker build seems to fail without this?
 		dockerUseBuildKit: true,
 		dockerfileContents: strings.Join([]string{
 			"FROM mirror.gcr.io/busybox",
 			"USER 10",
 			"RUN --mount=type=cache,uid=10,target=/cache touch /cache/10.txt",
 			"USER 0",
 			"RUN --mount=type=cache,target=/cache touch /cache/0.txt",
+			"RUN --mount=type=cache,uid=10,target=/cache touch /cache/0+10.txt",
 			"RUN mkdir -m 770 /results /results/0 /results/10 /results/0+10",
 			"RUN chown -R 10 /results",
-			"RUN --mount=type=cache,target=/cache cp -a /cache/* /results/0",
+			"RUN --mount=type=cache,target=/cache cp -av /cache/* /results/0",
 			"USER 10",
-			"RUN --mount=type=cache,uid=10,target=/cache cp -a /cache/* /results/10",
+			"RUN --mount=type=cache,uid=10,target=/cache cp -av /cache/* /results/10",
 			"USER 0",
-			"RUN --mount=type=cache,uid=10,target=/cache cp -a /cache/* /results/0+10",
+			"RUN --mount=type=cache,uid=10,target=/cache cp -av /cache/* /results/0+10",
 			"RUN touch -r /bin `find /results -print`",
 		}, "\n"),
 	},
@@ -3895,12 +3897,12 @@ var internalTestCases = []testCase{
 }
 
 func TestCommit(t *testing.T) {
-	t.Parallel()
 	testCases := []struct {
 		description             string
 		baseImage               string
 		changes, derivedChanges []string
 		config, derivedConfig   *docker.Config
+		dontParallelize         bool // uses shared state that lives elsewhere
 	}{
 		{
 			description: "defaults",
@@ -4239,16 +4241,20 @@ func TestCommit(t *testing.T) {
 	}
 	store, err := storage.GetStore(options)
 	require.NoErrorf(t, err, "error creating buildah storage at %q", rootDir)
-	defer func() {
+	t.Cleanup(func() {
 		if store != nil {
 			_, err := store.Shutdown(true)
 			require.NoErrorf(t, err, "error shutting down storage for buildah")
 		}
-	}()
+	})
 
 	// walk through test cases
 	for testIndex, testCase := range testCases {
 		t.Run(testCase.description, func(t *testing.T) {
+			if !testCase.dontParallelize {
+				t.Parallel()
+			}
+
 			test := testCases[testIndex]
 
 			// create the test container, then commit it, using the docker client

tests/conformance/testdata/Dockerfile.edgecases

@@ -1,4 +1,4 @@
-# Note: Hopefully a registries.conf alias redirects this to quay.io/libpod/mirror.gcr.io/busybox
+# Note: Hopefully a registries.conf alias redirects this to quay.io/libpod/busybox
 FROM mirror.gcr.io/busybox
 
 MAINTAINER docker <docker@docker.io>

tests/from.bats

@@ -176,7 +176,7 @@ load helpers
   skip_if_no_runtime
 
   _prefetch alpine
-  run_buildah from --quiet --cpu-period=5000 --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --cpu-period=5000 --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run $cid /bin/sh -c "cut -d ' ' -f 2 /sys/fs/cgroup/\$(awk -F: '{print \$NF}' /proc/self/cgroup)/cpu.max"
   expect_output "5000"
@@ -201,7 +201,7 @@ load helpers
 
   _prefetch alpine
   for shares in 2 200 2000 12345 20000 200000 ; do
-    run_buildah from --quiet --cpu-shares=${shares} --pull $WITH_POLICY_JSON alpine
+    run_buildah from --quiet --cpu-shares=${shares} --pull=false $WITH_POLICY_JSON alpine
     cid=$output
     local converted="$(convert_v1_shares_to_v2_weight ${shares})"
     local expect="(weight ${converted##* }|weight ${converted%% *})"
@@ -229,7 +229,7 @@ load helpers
   skip_if_no_runtime
 
   _prefetch alpine
-  run_buildah from --quiet --cpuset-mems=0 --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --cpuset-mems=0 --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run $cid /bin/sh -c "cat /sys/fs/cgroup/\$(awk -F : '{print \$NF}' /proc/self/cgroup)/cpuset.mems"
   expect_output "0"
@@ -254,7 +254,7 @@ load helpers
   skip_if_no_runtime
 
   _prefetch alpine
-  run_buildah from --quiet --volume=${TEST_SCRATCH_DIR}:/myvol --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --volume=${TEST_SCRATCH_DIR}:/myvol --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run $cid -- cat /proc/mounts
   expect_output --substring " /myvol "
@@ -312,7 +312,7 @@ load helpers
   skip_if_no_runtime
 
   _prefetch alpine
-  run_buildah from --quiet --shm-size=80m --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --shm-size=80m --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run $cid -- df -h /dev/shm
   expect_output --substring " 80.0M "
@@ -322,7 +322,7 @@ load helpers
   skip_if_no_runtime
 
   _prefetch alpine
-  run_buildah from --quiet --add-host=localhost:127.0.0.1 --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --add-host=localhost:127.0.0.1 --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah run --net=container $cid -- cat /etc/hosts
   expect_output --substring "127.0.0.1[[:blank:]]*localhost"
@@ -331,7 +331,7 @@ load helpers
 @test "from name test" {
   _prefetch alpine
   container_name=mycontainer
-  run_buildah from --quiet --name=${container_name} --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --name=${container_name} --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah inspect --format '{{.Container}}' ${container_name}
 }

tests/inspect.bats

@@ -15,7 +15,7 @@ load helpers
 
 @test "inspect" {
   _prefetch alpine
-  run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah commit $WITH_POLICY_JSON "$cid" alpine-image
 

tests/push.bats

@@ -89,7 +89,7 @@ load helpers
 
 @test "push should fail with nonexistent authfile" {
   _prefetch alpine
-  run_buildah from --quiet --pull $WITH_POLICY_JSON alpine
+  run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
   cid=$output
   run_buildah images -q
   imageid=$output
@@ -216,7 +216,7 @@ load helpers
 
 @test "push with --compression-format" {
   _prefetch alpine
-  run_buildah from --quiet --pull alpine
+  run_buildah from --quiet --pull=false alpine
   cid=$output
   run_buildah images -q
   imageid=$output

tests/subscriptions.bats

@@ -41,7 +41,7 @@ load helpers
     # setup the test container
     _prefetch alpine
     run_buildah --default-mounts-file "$MOUNTS_PATH" \
-		from --quiet --pull $WITH_POLICY_JSON alpine
+		from --quiet --pull=false $WITH_POLICY_JSON alpine
     cid=$output
 
     # test a standard mount to /run/secrets