SREP-1881 add oc to cad (#620)

* add OC cli to CAD image and adjust cpu and memory requests

* SREP-1881: add oc client wrapper to allow usage of oc in CAD

Author: rolandmkunkel

build/Dockerfile

@@ -17,6 +17,13 @@ ARG VERSION
 ARG VCS_REF
 ARG DOCKERFILE_PATH
 
+RUN microdnf update -y && \
+    microdnf install wget tar gzip -y
+RUN wget https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz -O /tmp/oc.tar.gz && \
+    tar -zxvf /tmp/oc.tar.gz -C /tmp && \
+    mv /tmp/oc /bin/oc && \
+    rm -f /tmp/oc.tar.gz
+
 LABEL vendor="RedHat" \
   name="openshift/configuration-anomaly-detection" \
   description="a CLI tool to detect and mitigate configuration mishaps" \

cadctl/cmd/investigate/investigate.go

@@ -28,7 +28,6 @@ import (
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/ccam"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/precheck"
-	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
 	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"github.com/openshift/configuration-anomaly-detection/pkg/managedcloud"
 	"github.com/openshift/configuration-anomaly-detection/pkg/metrics"
@@ -75,9 +74,6 @@ func run(_ *cobra.Command, _ []string) error {
 		return fmt.Errorf("missing required environment variable BACKPLANE_URL")
 	}
 
-	// Set k8s environment configuration for this session
-	k8sclient.SetBackplaneURL(backplaneURL)
-
 	// Load managedcloud environment variables
 	backplaneInitialARN := os.Getenv("BACKPLANE_INITIAL_ARN")
 	if backplaneInitialARN == "" {
@@ -154,7 +150,7 @@ func run(_ *cobra.Command, _ []string) error {
 		return fmt.Errorf("could not initialize backplane client: %w", err)
 	}
 
-	builder, err := investigation.NewResourceBuilder(pdClient, ocmClient, bpClient, clusterID, alertInvestigation.Name(), logLevelFlag, pipelineNameEnv)
+	builder, err := investigation.NewResourceBuilder(pdClient, ocmClient, bpClient, clusterID, alertInvestigation.Name(), logLevelFlag, pipelineNameEnv, backplaneURL)
 	if err != nil {
 		return fmt.Errorf("failed to create resource builder: %w", err)
 	}
@@ -164,13 +160,21 @@ func run(_ *cobra.Command, _ []string) error {
 		// We ignore the error here because we just want to get any resources that were created.
 		resources, _ := builder.Build()
 
-		// Cleanup k8s client if it exists
-		if resources != nil && resources.K8sClient != nil {
-			// Failing the k8sclient cleanup call is not critical
+		// Cleanup rest config if it exists
+		if resources != nil && resources.RestConfig != nil {
+			// Failing the rest config cleanup call is not critical
 			// There is garbage collection for the RBAC within MCC https://issues.redhat.com/browse/OSD-27692
 			// We only log the error for now but could add it to the investigation notes or handle differently
-			logging.Info("Cleaning kube-api access")
-			deferErr := resources.K8sClient.Clean()
+			logging.Info("Cleaning cluster api access")
+			deferErr := resources.RestConfig.Clean()
+			if deferErr != nil {
+				logging.Error(deferErr)
+			}
+		}
+
+		if resources != nil && resources.OCClient != nil {
+			logging.Info("Cleaning oc kubeconfig file access")
+			deferErr := resources.OCClient.Clean()
 			if deferErr != nil {
 				logging.Error(deferErr)
 			}

openshift/template.yaml

@@ -339,8 +339,8 @@ objects:
           cpu: 100m
           memory: 256Mi
         requests:
-          cpu: 10m
-          memory: 64Mi
+          cpu: 20m
+          memory: 128Mi
 - apiVersion: batch/v1
   kind: CronJob
   metadata:

pkg/investigations/investigation/errors.go

@@ -27,18 +27,40 @@ type AWSClientError struct {
 	Err       error
 }
 
-func (a *AWSClientError) Unwrap() error { return a.Err }
+func (e AWSClientError) Unwrap() error { return e.Err }
 
 func (e AWSClientError) Error() string {
 	return fmt.Sprintf("could not retrieve aws credentials for %s: %s", e.ClusterID, e.Err.Error())
 }
 
+type RestConfigError struct {
+	ClusterID string
+	Err       error
+}
+
+func (e RestConfigError) Unwrap() error { return e.Err }
+
+func (e RestConfigError) Error() string {
+	return fmt.Sprintf("could not create rest config for %s: %s", e.ClusterID, e.Err.Error())
+}
+
+type OCClientError struct {
+	ClusterID string
+	Err       error
+}
+
+func (e OCClientError) Unwrap() error { return e.Err }
+
+func (e OCClientError) Error() string {
+	return fmt.Sprintf("could not create oc client for %s: %s", e.ClusterID, e.Err.Error())
+}
+
 type K8SClientError struct {
 	ClusterID string
 	Err       error
 }
 
-func (a *K8SClientError) Unwrap() error { return a.Err }
+func (e K8SClientError) Unwrap() error { return e.Err }
 
 func (e K8SClientError) Error() string {
 	return fmt.Sprintf("could not build k8s client for %s: %s", e.ClusterID, e.Err.Error())

pkg/investigations/investigation/investigation.go

@@ -1,15 +1,22 @@
 package investigation
 
 import (
+	"context"
+
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
-	"github.com/openshift/configuration-anomaly-detection/pkg/backplane"
+	"github.com/openshift/backplane-cli/pkg/cli/config"
+	bpremediation "github.com/openshift/backplane-cli/pkg/remediation"
+
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
+	"k8s.io/client-go/rest"
 
 	"github.com/openshift/configuration-anomaly-detection/pkg/aws"
+	"github.com/openshift/configuration-anomaly-detection/pkg/backplane"
 	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
 	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"github.com/openshift/configuration-anomaly-detection/pkg/managedcloud"
 	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"github.com/openshift/configuration-anomaly-detection/pkg/oc"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
 )
@@ -37,6 +44,7 @@ func NewResourceBuilder(
 	name string,
 	logLevel string,
 	pipelineName string,
+	backplaneUrl string,
 ) (ResourceBuilder, error) {
 	rb := &ResourceBuilderT{
 		buildLogger:  true,
@@ -45,6 +53,7 @@ func NewResourceBuilder(
 		logLevel:     logLevel,
 		pipelineName: pipelineName,
 		ocmClient:    ocmClient,
+		backplaneUrl: backplaneUrl,
 		builtResources: &Resources{
 			BpClient:  bpClient,
 			PdClient:  pdClient,
@@ -72,17 +81,21 @@ type Resources struct {
 	ClusterDeployment *hivev1.ClusterDeployment
 	AwsClient         aws.Client
 	BpClient          backplane.Client
+	RestConfig        *RestConfig
 	K8sClient         k8sclient.Client
 	OcmClient         ocm.Client
 	PdClient          pagerduty.Client
 	Notes             *notewriter.NoteWriter
+	OCClient          oc.Client
 }
 
 type ResourceBuilder interface {
 	WithCluster() ResourceBuilder
 	WithClusterDeployment() ResourceBuilder
 	WithAwsClient() ResourceBuilder
+	WithRestConfig() ResourceBuilder
 	WithK8sClient() ResourceBuilder
+	WithOC() ResourceBuilder
 	WithNotes() ResourceBuilder
 	Build() (*Resources, error)
 }
@@ -91,14 +104,17 @@ type ResourceBuilderT struct {
 	buildCluster           bool
 	buildClusterDeployment bool
 	buildAwsClient         bool
+	buildRestConfig        bool
 	buildK8sClient         bool
+	buildOC                bool
 	buildNotes             bool
 	buildLogger            bool
 
 	clusterId    string
 	name         string
 	logLevel     string
 	pipelineName string
+	backplaneUrl string
 
 	ocmClient *ocm.SdkClient
 
@@ -118,13 +134,26 @@ func (r *ResourceBuilderT) WithClusterDeployment() ResourceBuilder {
 	return r
 }
 
+func (r *ResourceBuilderT) WithRestConfig() ResourceBuilder {
+	r.WithCluster()
+	r.buildRestConfig = true
+	return r
+}
+
 func (r *ResourceBuilderT) WithAwsClient() ResourceBuilder {
 	r.WithCluster()
 	r.buildAwsClient = true
 	return r
 }
 
+func (r *ResourceBuilderT) WithOC() ResourceBuilder {
+	r.WithRestConfig()
+	r.buildOC = true
+	return r
+}
+
 func (r *ResourceBuilderT) WithK8sClient() ResourceBuilder {
+	r.WithRestConfig()
 	r.buildK8sClient = true
 	return r
 }
@@ -154,49 +183,107 @@ func (r *ResourceBuilderT) Build() (*Resources, error) {
 		}
 	}
 
-	// Dependent resources can only be built if a cluster object exists.
-	//nolint:nestif
-	if r.builtResources.Cluster != nil {
-		internalClusterId := r.builtResources.Cluster.ID()
-
-		if r.buildAwsClient && r.builtResources.AwsClient == nil {
-			r.builtResources.AwsClient, err = managedcloud.CreateCustomerAWSClient(r.builtResources.Cluster, r.ocmClient)
-			if err != nil {
-				r.buildErr = AWSClientError{ClusterID: r.clusterId, Err: err}
-				return r.builtResources, r.buildErr
-			}
+	if r.buildNotes && r.builtResources.Notes == nil {
+		r.builtResources.Notes = notewriter.New(r.name, logging.RawLogger)
+	}
+
+	internalClusterId := r.builtResources.Cluster.ID()
+
+	if r.buildAwsClient && r.builtResources.AwsClient == nil {
+		r.builtResources.AwsClient, err = managedcloud.CreateCustomerAWSClient(r.builtResources.Cluster, r.ocmClient)
+		if err != nil {
+			r.buildErr = AWSClientError{ClusterID: r.clusterId, Err: err}
+			return r.builtResources, r.buildErr
 		}
+	}
 
-		if r.buildK8sClient && r.builtResources.K8sClient == nil {
-			logging.Infof("creating k8s client for %s", r.name)
-			r.builtResources.K8sClient, err = k8sclient.New(r.builtResources.Cluster.ID(), r.ocmClient, r.name)
-			if err != nil {
-				r.buildErr = K8SClientError{ClusterID: r.clusterId, Err: err}
-				return r.builtResources, r.buildErr
-			}
+	if r.buildRestConfig && r.builtResources.RestConfig == nil {
+		r.builtResources.RestConfig, err = newRestConfig(r.builtResources.Cluster.ID(), r.backplaneUrl, r.ocmClient, r.name)
+		if err != nil {
+			r.buildErr = RestConfigError{ClusterID: r.clusterId, Err: err}
+			return r.builtResources, r.buildErr
 		}
+	}
+
+	if r.buildK8sClient && r.builtResources.K8sClient == nil {
+		logging.Infof("creating k8s client for %s", r.name)
+		r.builtResources.K8sClient, err = k8sclient.New(&r.builtResources.RestConfig.Config)
+		if err != nil {
+			r.buildErr = K8SClientError{ClusterID: r.clusterId, Err: err}
+			return r.builtResources, r.buildErr
+		}
+	}
 
-		if r.buildClusterDeployment && r.builtResources.ClusterDeployment == nil {
-			r.builtResources.ClusterDeployment, err = r.ocmClient.GetClusterDeployment(internalClusterId)
-			if err != nil {
-				r.buildErr = ClusterDeploymentNotFoundError{ClusterID: r.clusterId, Err: err}
-				return r.builtResources, r.buildErr
-			}
+	if r.buildOC && r.builtResources.OCClient == nil {
+		r.builtResources.OCClient, err = oc.New(context.Background(), &r.builtResources.RestConfig.Config)
+		if err != nil {
+			r.buildErr = OCClientError{ClusterID: r.clusterId, Err: err}
+			return r.builtResources, r.buildErr
 		}
+	}
 
-		if r.buildLogger {
-			// Re-initialize the logger with the cluster ID.
-			logging.RawLogger = logging.InitLogger(r.logLevel, r.pipelineName, internalClusterId)
+	if r.buildClusterDeployment && r.builtResources.ClusterDeployment == nil {
+		r.builtResources.ClusterDeployment, err = r.ocmClient.GetClusterDeployment(internalClusterId)
+		if err != nil {
+			r.buildErr = ClusterDeploymentNotFoundError{ClusterID: r.clusterId, Err: err}
+			return r.builtResources, r.buildErr
 		}
 	}
 
-	if r.buildNotes && r.builtResources.Notes == nil {
-		r.builtResources.Notes = notewriter.New(r.name, logging.RawLogger)
+	if r.buildLogger {
+		// Re-initialize the logger with the cluster ID.
+		logging.RawLogger = logging.InitLogger(r.logLevel, r.pipelineName, internalClusterId)
 	}
 
 	return r.builtResources, nil
 }
 
+type remediationCleaner struct {
+	clusterID             string
+	ocmClient             ocm.Client
+	remediationInstanceId string
+	backplaneUrl          string
+}
+
+type Cleaner interface {
+	Clean() error
+}
+
+type RestConfig struct {
+	rest.Config
+	backplaneUrl string
+	Cleaner
+}
+
+func (cleaner remediationCleaner) Clean() error {
+	return deleteRemediation(cleaner.clusterID, cleaner.backplaneUrl, cleaner.ocmClient, cleaner.remediationInstanceId)
+}
+
+// New returns a k8s rest config for the given cluster scoped to a given remediation's permissions.
+func newRestConfig(clusterID, backplaneUrl string, ocmClient ocm.Client, remediationName string) (*RestConfig, error) {
+	decoratedCfg, remediationInstanceId, err := bpremediation.CreateRemediationWithConn(
+		config.BackplaneConfiguration{URL: backplaneUrl},
+		ocmClient.GetConnection(),
+		clusterID,
+		remediationName,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RestConfig{*decoratedCfg, backplaneUrl, remediationCleaner{clusterID, ocmClient, remediationInstanceId, backplaneUrl}}, nil
+}
+
+// Cleanup removes the remediation created for the cluster.
+func deleteRemediation(clusterID, backplaneUrl string, ocmClient ocm.Client, remediationInstanceId string) error {
+	return bpremediation.DeleteRemediationWithConn(
+		config.BackplaneConfiguration{URL: backplaneUrl},
+		ocmClient.GetConnection(),
+		clusterID,
+		remediationInstanceId,
+	)
+}
+
 // This is an implementation to be used in tests, but putting it into a _test.go file will make it not resolvable.
 type ResourceBuilderMock struct {
 	Resources  *Resources
@@ -215,6 +302,14 @@ func (r *ResourceBuilderMock) WithAwsClient() ResourceBuilder {
 	return r
 }
 
+func (r *ResourceBuilderMock) WithRestConfig() ResourceBuilder {
+	return r
+}
+
+func (r *ResourceBuilderMock) WithOC() ResourceBuilder {
+	return r
+}
+
 func (r *ResourceBuilderMock) WithNotes() ResourceBuilder {
 	return r
 }