feat(testing): initial integration test implementation (#184)

This commit uses container image to run integration tests. The
tests are automated and run on a K3s installation. Integration
tests are triggered as a make target.

e.g. make integration-test-suite

This approach provides the code coverage report in suite's run
logs.

This change fixes a labelling bug where resources that are not
supposed to be labelled are labelled.

GitHub actions has been modifed to accommodate integration testing.

Signed-off-by: AmitKumarDas <[email protected]>

.github/workflows/test-release.yaml

@@ -16,6 +16,17 @@ jobs:
         with:
           go-version: 1.13.5
       - run: make ${{ matrix.test }}
+  integrationtest:
+    runs-on: ubuntu-18.04
+    name: integrationtest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Setup Golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.13.5
+      - run: sudo make integration-test-suite
   e2etest:
     runs-on: ubuntu-18.04
     strategy:
@@ -33,7 +44,7 @@ jobs:
   release:
     name: Make Github Release
     runs-on: ubuntu-18.04
-    needs: ['unittest', 'e2etest']
+    needs: ['unittest', 'e2etest', 'integrationtest']
     steps:
       - name: Checkout Code
         uses: actions/checkout@v1

.gitignore

@@ -4,6 +4,7 @@ d-operators
 test/bin/
 test/kubebin/
 test/e2e/uninstall-k3s.txt
+test/integration/uninstall-k3s.txt
 uninstall-k3s.txt
 dope
 daction

Dockerfile.testing

@@ -0,0 +1,9 @@
+# --------------------------
+# Build d-operators binary
+# --------------------------
+FROM golang:1.13.5 as builder
+
+WORKDIR /mayadata.io/d-operators/
+
+# copy all manifests
+COPY . .

Makefile

@@ -34,10 +34,18 @@ test:
 testv:
 	@go test ./... -cover -v -args --logtostderr -v=2
 
+.PHONY: integration-test
+integration-test: 
+	@go test ./... -cover --tags=integration
+
 .PHONY: e2e-test
 e2e-test: 
 	@cd test/e2e && ./suite.sh
 
+.PHONY: integration-test-suite
+integration-test-suite:
+	@cd test/integration && ./suite.sh
+
 .PHONY: image
 image:
 	docker build -t $(IMG_REPO):$(PACKAGE_VERSION) -t $(IMG_REPO):latest .

common/labels/labels_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

common/metac/hook_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

common/unstruct/list_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

common/unstruct/unstruct_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/command/reconciler_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/http/reconciler_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/assert_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/create_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/createordelete_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/delete_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/run_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/task_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

controller/run/update_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

pkg/command/job_builder_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

pkg/command/reconciler_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

pkg/kubernetes/utility.go

@@ -25,9 +25,7 @@ import (
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/klog/v2"
 
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"openebs.io/metac/dynamic/clientset"
 	dynamicclientset "openebs.io/metac/dynamic/clientset"
 	dynamicdiscovery "openebs.io/metac/dynamic/discovery"
@@ -38,7 +36,6 @@ type UtilityConfig struct {
 	KubeConfig   *rest.Config
 	APIDiscovery *dynamicdiscovery.APIResourceDiscovery
 	Retry        *Retryable
-	IsTeardown   bool
 }
 
 // UtilityFuncs exposes Utility fields as functional options
@@ -72,14 +69,6 @@ type Utility struct {
 	// definitions aka CRDs
 	crdClient apiextnv1beta1.ApiextensionsV1beta1Interface
 
-	// If resources created using this utility should be deleted
-	// when this instance's Teardown method is invoked
-	isTeardown bool
-
-	// list of teardown functions invoked when this instance's
-	// Teardown method is invoked
-	teardownFuncs []func() error
-
 	// error as value
 	err error
 }
@@ -136,7 +125,6 @@ func NewUtility(config UtilityConfig) (*Utility, error) {
 	// 	Following order needs to be maintained
 	var setters = []func(UtilityConfig){
 		// pre settings
-		u.setTeardownFlag,
 		u.setAPIResourceDiscoveryOrDefault,
 
 		// post settings
@@ -153,10 +141,6 @@ func NewUtility(config UtilityConfig) (*Utility, error) {
 	return u, nil
 }
 
-func (u *Utility) setTeardownFlag(config UtilityConfig) {
-	u.isTeardown = config.IsTeardown
-}
-
 func (u *Utility) setAPIResourceDiscoveryOrDefault(config UtilityConfig) {
 	u.apiResourceDiscovery = config.APIDiscovery
 	if u.apiResourceDiscovery != nil {
@@ -205,64 +189,6 @@ func (u *Utility) setKubeClientset(config UtilityConfig) {
 	)
 }
 
-// MustTeardown deletes resources created through this instance
-func (u *Utility) MustTeardown() {
-	// cleanup in descending order
-	for i := len(u.teardownFuncs) - 1; i >= 0; i-- {
-		teardown := u.teardownFuncs[i]
-		err := teardown()
-		if err != nil {
-			if apierrors.IsNotFound(err) {
-				klog.V(3).Infof(
-					"Teardown ignored: Resource not found: %+v",
-					err,
-				)
-				continue
-			}
-			if apierrors.IsConflict(err) {
-				klog.V(3).Infof(
-					"Teardown ignored: Conflict: %+v",
-					err,
-				)
-				continue
-			}
-			// we treat the teardown error as level 1 Info
-			klog.V(1).Infof(
-				"Teardown failed: %s: %+v",
-				apierrors.ReasonForError(err),
-				err,
-			)
-		}
-	}
-}
-
-// Teardown optionally deletes resources created through this
-// instance
-func (u *Utility) Teardown() {
-	if !u.isTeardown {
-		return
-	}
-	u.MustTeardown()
-}
-
-// MustAddToTeardown adds the given teardown function to
-// the list of teardown functions
-func (u *Utility) MustAddToTeardown(teardown func() error) {
-	if teardown == nil {
-		return
-	}
-	u.teardownFuncs = append(u.teardownFuncs, teardown)
-}
-
-// AddToTeardown optionally adds the given teardown function to
-// the list of teardown functions
-func (u *Utility) AddToTeardown(teardown func() error) {
-	if !u.isTeardown {
-		return
-	}
-	u.MustAddToTeardown(teardown)
-}
-
 // GetClientForAPIVersionAndKind returns the dynamic client for the
 // given api version & kind
 func (u *Utility) GetClientForAPIVersionAndKind(

pkg/kubernetes/utility_int_test.go

@@ -0,0 +1,49 @@
+// +build integration
+
+/*
+Copyright 2020 The MayaData Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"testing"
+
+	"k8s.io/client-go/rest"
+)
+
+func TestNewFixtureIT(t *testing.T) {
+	var tests = map[string]struct {
+		kubeConfig *rest.Config
+		isErr      bool
+	}{
+		"nil kubeconfig": {},
+	}
+	for name, mock := range tests {
+		name := name
+		mock := mock
+		t.Run(name, func(t *testing.T) {
+			_, err := NewUtility(UtilityConfig{
+				KubeConfig: mock.kubeConfig,
+			})
+			if mock.isErr && err == nil {
+				t.Fatal("Expected error got none")
+			}
+			if !mock.isErr && err != nil {
+				t.Fatalf("Expected no error got %s", err.Error())
+			}
+		})
+	}
+}

pkg/kubernetes/utility_test.go

@@ -1,3 +1,5 @@
+// +build !integration
+
 /*
 Copyright 2020 The MayaData Authors.
 

pkg/recipe/assert.go

@@ -158,5 +158,17 @@ func (a *Assertable) Run() (types.AssertStatus, error) {
 			return types.AssertStatus{}, a.err
 		}
 	}
+	var errorOnAssertFailure bool
+	if a.Assert.ErrorOnAssertFailure != nil {
+		errorOnAssertFailure = *a.Assert.ErrorOnAssertFailure
+	}
+	if errorOnAssertFailure &&
+		a.status.Phase != types.AssertResultPassed {
+		return types.AssertStatus{}, errors.Errorf(
+			"%s: ErrorOnAssertFailure=%t",
+			a.status,
+			errorOnAssertFailure,
+		)
+	}
 	return *a.status, nil
 }