chore(QA-148): CLI acceptance tests coverage

This commit have no practical implications on the test coverage, but it
aligns the acceptance testing environment with the rest of the backend
components.

changelog: none

Signed-off-by: Alf-Rune Siqveland <[email protected]>

Author: alfrunes

.gitlab-ci.yml

@@ -29,7 +29,6 @@ include:
     file: '.gitlab-ci-check-docker-build.yml'
 
 test:unit:
-  image: golang:1.15
   needs: []
   services:
     - name: mongo:4.4
@@ -44,17 +43,21 @@ test:acceptance:
     - /^saas-[a-zA-Z0-9.-]+$/
   tags:
     - docker
-  image: docker:19.03.13
+  image: docker:20.10.12
   services:
-    - name: docker:19.03.13-dind
+    - name: docker:20.10.12-dind
       alias: docker
   before_script:
     - apk add docker-compose make
   script:
     - make acceptance-tests
   after_script:
-    - make acceptance-tests-logs
-    - make acceptance-tests-down
+    - set -- tests/coverage-acceptance@*.txt
+    - head -n 1 $1 > tests/coverage-acceptance.txt
+    - |
+      for cover in $@; do
+        tail -n +2 $cover >> tests/coverage-acceptance.txt;
+      done
   artifacts:
     expire_in: 2w
     paths:
@@ -66,7 +69,7 @@ publish:acceptance:
   stage: publish
   except:
     - /^saas-[a-zA-Z0-9.-]+$/
-  image: golang:1.15
+  image: golang:1.17
   needs:
     - job: test:acceptance
       artifacts: true
@@ -80,7 +83,6 @@ publish:acceptance:
     #  and pass few others as command line arguments.
     #  See also https://docs.coveralls.io/api-reference
     - export CI_BRANCH=${CI_COMMIT_BRANCH}
-    - export CI_PR_NUMBER=${CI_COMMIT_BRANCH#pr_}
   script:
     - goveralls
       -repotoken ${COVERALLS_TOKEN}

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.5-alpine3.12 as builder
+FROM golang:1.17.6-alpine3.15 as builder
 RUN apk add --no-cache \
     xz-dev \
     musl-dev \

Dockerfile.acceptance

@@ -13,11 +13,9 @@ FROM alpine:3.15.0
 RUN apk add --no-cache ca-certificates xz
 RUN mkdir -p /etc/deviceconnect
 COPY ./config.yaml /etc/deviceconnect
-COPY --from=builder /go/src/github.com/mendersoftware/deviceconnect/bin/deviceconnect.test /usr/bin
-ENTRYPOINT ["/usr/bin/deviceconnect.test", \
-        "-test.coverprofile=/testing/coverage-acceptance.txt", \
-        "-acceptance-testing", \
-        "--", "--config=/etc/deviceconnect/config.yaml", \
+COPY --from=builder /go/src/github.com/mendersoftware/deviceconnect/bin/deviceconnect.test /usr/bin/deviceconnect
+ENTRYPOINT ["/usr/bin/deviceconnect", \
+        "--config=/etc/deviceconnect/config.yaml", \
         "server", "--automigrate"]
 
 EXPOSE 8080

Makefile

@@ -37,9 +37,8 @@ $(BINFILE): $(SRCFILES)
 	$(GO) build -o $@ .
 
 $(BINFILE).test: $(GOFILES)
-	go test -c -o $(BINFILE).test \
-		-cover -covermode atomic \
-		-coverpkg $(PACKAGES)
+	go test -c -o $(BINFILE).test -tags main \
+		-cover -covermode atomic -coverpkg $(PACKAGES)
 
 $(COVERFILE): $(GOFILES)
 	$(GO) test -cover -covermode=atomic -coverprofile=$@ ${TEST_FLAGS} ./...
@@ -70,26 +69,28 @@ docker: bin/deviceconnect.docker
 .PHONY: docker-test
 docker-test: bin/deviceconnect.acceptance.docker
 
-.PHONY: acceptance-tests
-acceptance-tests: docker-test docs
+.PHONY: acceptance-tests-run
+acceptance-tests-run: docker-test docs
 	docker-compose \
-		-f tests/docker-compose-acceptance.yml \
 		-p acceptance \
-		up -d
-	docker attach acceptance_acceptance_1
+		-f tests/docker-compose.yml \
+		run tester
 
 .PHONY: acceptance-tests-logs
-acceptance-tests-logs:
-	for service in $(shell docker-compose -f tests/docker-compose-acceptance.yml -p acceptance ps -a --services); do \
-		docker-compose -p acceptance -f tests/docker-compose-acceptance.yml \
+acceptance-tests-logs: acceptance-tests-run
+	for service in $(shell docker-compose -f tests/docker-compose.yml -p acceptance ps -a --services); do \
+		docker-compose -p acceptance -f tests/docker-compose.yml \
 				logs --no-color $$service > "tests/acceptance.$${service}.logs"; \
 	done
 
 .PHONY: acceptance-tests-down
 acceptance-tests-down:
 	docker-compose \
-		-f tests/docker-compose-acceptance.yml \
-		-p acceptance down
+		-f tests/docker-compose.yml \
+		-p acceptance down --remove-orphans
+
+.PHONY: acceptance-tests
+acceptance-tests: acceptance-tests-logs acceptance-tests-down
 
 
 .PHONY: fmt

main_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Northern.tech AS
+// Copyright 2022 Northern.tech AS
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -12,39 +12,74 @@
 //    See the License for the specific language governing permissions and
 //    limitations under the License.
 
+//go:build main
+// +build main
+
 package main
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"flag"
+	"fmt"
+	"hash/crc64"
+	"io"
 	"os"
+	"os/signal"
 	"testing"
-)
 
-var (
-	acceptanceTesting bool
+	"github.com/mendersoftware/go-lib-micro/log"
 )
 
 func init() {
-	flag.BoolVar(&acceptanceTesting, "acceptance-testing", false,
-		"Acceptance testing mode, starts the application main function "+
-			"with cover mode enabled. Non-flag arguments are passed"+
-			"to the main application, add '--' after test flags to"+
-			"pass flags to main.",
-	)
+	// Make sure main does not exit before we have gathered coverage.
+	log.Log.ExitFunc = func(int) {}
 }
 
+const (
+	coverName = "coverage-acceptance"
+	coverExt  = ".txt"
+)
+
+var stdout = os.Stdout
+
 func TestMain(m *testing.M) {
-	flag.Parse()
-	if acceptanceTesting {
-		// Override 'run' flags to only execute TestDoMain
-		flag.Set("test.run", "TestDoMain")
+	argHash := crc64.New(crc64.MakeTable(crc64.ECMA))
+	for _, arg := range os.Args {
+		_, _ = argHash.Write([]byte(arg))
 	}
-	os.Exit(m.Run())
+	var b [6]byte
+	_, err := io.ReadFull(rand.Reader, b[:])
+	if err != nil {
+		panic(err)
+	}
+	// filename = "{coverName}@{hash(args)}-{48-bit rand}.txt"
+	fileNameCover := fmt.Sprintf("%s@%s-%s%s",
+		coverName,
+		hex.EncodeToString(argHash.Sum(nil)),
+		hex.EncodeToString(b[:]),
+		coverExt,
+	)
+
+	// Override arguments passed to "testing" package
+	os.Args = os.Args[:1]
+	flag.Set("test.run", "TestRunMain")
+	flag.Set("test.coverprofile", fileNameCover)
+
+	// Run tests
+	exitCode := m.Run()
+	os.Stdout = stdout
+	os.Exit(exitCode)
 }
 
-func TestDoMain(t *testing.T) {
-	if !acceptanceTesting {
-		t.Skip()
-	}
-	doMain(append(os.Args[:1], flag.Args()...))
+func TestRunMain(t *testing.T) {
+	stopChan := make(chan os.Signal, 1)
+	signal.Notify(stopChan, os.Interrupt)
+	go func() {
+		doMain(os.Args[:cap(os.Args)])
+		stopChan <- os.Interrupt
+	}()
+	<-stopChan
+	// Prevent the output from testing to hit stdout
+	os.Stdout = os.Stderr
 }

tests/Dockerfile

@@ -0,0 +1,4 @@
+FROM mendersoftware/mender-test-containers:acceptance-testing
+
+# FIXME: The old and deprecated version of msgpack is a dependency of bravado (unused)
+RUN pip uninstall -y msgpack && pip install msgpack==1.0.3

tests/docker-compose-acceptance.yml tests/docker-compose.yml

@@ -1,24 +1,34 @@
 version: '2.1'
 services:
 
-    acceptance:
-        image: mendersoftware/mender-test-containers:acceptance-testing
+    tester:
+        build:
+          context: "."
+          dockerfile: Dockerfile
+        image: mendersoftware/mender-test-containers:acceptance-testing-deviceconnect
         networks:
             - mender
         volumes:
             - ".:/testing"
+            - "/var/run/docker.sock:/var/run/docker.sock"
         depends_on:
             - mender-deviceconnect
+            - mender-mongo
+            - mender-nats
             - mmock
 
     mender-deviceconnect:
+      build:
+        dockerfile: "Dockerfile.acceptance"
+        context: ".."
       image: mendersoftware/deviceconnect:prtest
       networks:
         mender:
           aliases:
             - mender-deviceconnect
       volumes:
         - ".:/testing"
+      working_dir: /testing
       depends_on:
         - mender-mongo
         - mender-nats

tests/run.sh

@@ -9,8 +9,6 @@ export TESTING_MMOCK_HOST=${TESTING_MMOCK_HOST:="mmock:8080"}
 
 export PYTHONDONTWRITEBYTECODE=1
 
-pip3 install --quiet --force-reinstall -r requirements.txt 
-
 # if we're running in a container, wait a little before starting tests
 [ $$ -eq 1 ] && sleep 10