Skip to content

Commit

Permalink
Merge pull request #333 from bergmannf/osd-15156-health-gcp
Browse files Browse the repository at this point in the history
[OSD-15156] Add gcp support to cluster health
  • Loading branch information
openshift-merge-robot authored Mar 1, 2023
2 parents 8ccd2f6 + 658b97f commit 5078885
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 41 deletions.
148 changes: 107 additions & 41 deletions cmd/cluster/health.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cluster

import (
"errors"
"fmt"
"log"
"strconv"
Expand All @@ -11,6 +12,7 @@ import (
"github.com/openshift/osdctl/pkg/osdCloud"
"github.com/openshift/osdctl/pkg/utils"
"github.com/spf13/cobra"
"google.golang.org/api/iterator"
"gopkg.in/yaml.v2"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
)
Expand Down Expand Up @@ -96,59 +98,123 @@ func (o *healthOptions) run() error {
healthObject.Expected.Worker = int(cluster.Nodes().Compute())
}

awsClient, err := osdCloud.GenerateAWSClientForCluster(o.awsProfile, o.clusterID)
if err != nil {
return err
}

instances, err := awsClient.DescribeInstances(&ec2.DescribeInstancesInput{})
if err != nil {
return err
}
runningMasters := 0
runningInfra := 0
runningWorkers := 0
totalStopped := 0
totalCluster := 0

//Here we count the number of customer's running worker, infra and master instances in the cluster in the given region. To decide if the instance belongs to the cluster we are checking the Name Tag on the instance.
for idx := range instances.Reservations {
for _, inst := range instances.Reservations[idx].Instances {
tags := inst.Tags
for _, t := range tags {
if *t.Key == "Name" {
if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "master") {
totalCluster += 1
if *inst.State.Name == "running" {
runningMasters += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}

} else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "infra") {
totalCluster += 1
if *inst.State.Name == "running" {
runningInfra += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}
} else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "worker") {
totalCluster += 1
if *inst.State.Name == "running" {
runningWorkers += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}

if cluster.CloudProvider().ID() == "gcp" {
clusterResources, err := ocmClient.ClustersMgmt().V1().Clusters().Cluster(o.clusterID).Resources().Live().Get().Send()
if err != nil {
return err
}
projectClaimRaw, found := clusterResources.Body().Resources()["gcp_project_claim"]
if !found {
return fmt.Errorf("The gcp_project_claim was not found in the ocm resource")
}
projectClaim, err := osdCloud.ParseGcpProjectClaim(projectClaimRaw)
if err != nil {
log.Printf("Unmarshalling GCP projectClaim failed: %v\n", err)
return err
}
projectId := projectClaim.Spec.GcpProjectID
zones := cluster.Nodes().AvailabilityZones()
if projectId == "" || len(zones) == 0 {
return fmt.Errorf("ProjectID or Zones empty - aborting")
}
gcpClient, err := osdCloud.GenerateGCPComputeInstancesClient()
defer gcpClient.Close()
if err != nil {
return err
}
ownedLabel := "kubernetes-io-cluster-" + cluster.InfraID()
for _, zone := range zones {
instances := osdCloud.ListInstances(gcpClient, projectId, zone)
for {
instance, err := instances.Next()
if err == iterator.Done {
break
}
if err != nil {
return err
}
name := instance.GetName()
state := instance.GetStatus()
labels := instance.GetLabels()
belongsToCluster := false
for label := range labels {
if label == ownedLabel {
belongsToCluster = true
}
}
if !belongsToCluster {
log.Printf("Skipping a machine not belonging to the cluster: %s\n", name)
continue
}
totalCluster += 1
if state != "RUNNING" {
totalStopped += 1
} else {
if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "master") {
runningMasters += 1
} else if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "infra") {
runningInfra += 1
} else if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "worker") {
runningWorkers += 1
}
}
}
}
} else if cluster.CloudProvider().ID() == "aws" {
awsClient, err := osdCloud.GenerateAWSClientForCluster(o.awsProfile, o.clusterID)
if err != nil {
return err
}

instances, err := awsClient.DescribeInstances(&ec2.DescribeInstancesInput{})
if err != nil {
return err
}

//Here we count the number of customer's running worker, infra and master instances in the cluster in the given region. To decide if the instance belongs to the cluster we are checking the Name Tag on the instance.
for idx := range instances.Reservations {
for _, inst := range instances.Reservations[idx].Instances {
tags := inst.Tags
for _, t := range tags {
if *t.Key == "Name" {
if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "master") {
totalCluster += 1
if *inst.State.Name == "running" {
runningMasters += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}

} else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "infra") {
totalCluster += 1
if *inst.State.Name == "running" {
runningInfra += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}
} else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "worker") {
totalCluster += 1
if *inst.State.Name == "running" {
runningWorkers += 1
}
if *inst.State.Name == "stopped" {
totalStopped += 1
}
}
}
}
}
}
} else {
return errors.New(fmt.Sprintf("Unknown cloud provider found: %s", cluster.CloudProvider().ID()))
}

healthObject.Actual.Stopped = totalStopped
Expand Down
8 changes: 8 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/openshift/osdctl
go 1.19

require (
cloud.google.com/go/compute v1.7.0
github.com/PagerDuty/go-pagerduty v1.5.1
github.com/andygrunwald/go-jira v1.16.0
github.com/aws/aws-sdk-go v1.44.66
Expand All @@ -29,6 +30,8 @@ require (
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.13.0
google.golang.org/api v0.84.0
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.24.3
k8s.io/apimachinery v0.24.3
Expand Down Expand Up @@ -75,6 +78,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
Expand All @@ -83,6 +87,8 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa // indirect
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
Expand Down Expand Up @@ -133,6 +139,7 @@ require (
github.com/trivago/tgo v1.0.7 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a // indirect
go.opencensus.io v0.23.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/net v0.4.0 // indirect
Expand All @@ -142,6 +149,7 @@ require (
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW
cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8=
cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
Expand All @@ -41,6 +42,7 @@ cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJW
cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk=
cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
Expand Down Expand Up @@ -330,6 +332,7 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
Expand Down Expand Up @@ -420,13 +423,15 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa h1:7MYGT2XEMam7Mtzv1yDUYXANedWvwk3HKkR3MyGowy8=
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
Expand Down Expand Up @@ -930,6 +935,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
Expand Down Expand Up @@ -1367,6 +1373,7 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg=
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down Expand Up @@ -1459,6 +1466,7 @@ google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP
google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs=
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
Expand Down Expand Up @@ -1495,6 +1503,7 @@ google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
Expand Down
40 changes: 40 additions & 0 deletions pkg/osdCloud/gcp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package osdCloud

import (
"context"
"encoding/json"

compute "cloud.google.com/go/compute/apiv1"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
)

type GcpProjectClaimSpec struct {
GcpProjectID string `json:"gcpProjectID"`
}
type GcpProjectClaim struct {
Spec GcpProjectClaimSpec `json:"spec"`
}

func ParseGcpProjectClaim(raw string) (*GcpProjectClaim, error) {
var projectClaim GcpProjectClaim
err := json.Unmarshal([]byte(raw), &projectClaim)
if err != nil {
return nil, err
}
return &projectClaim, nil
}

func GenerateGCPComputeInstancesClient() (*compute.InstancesClient, error) {
ctx := context.Background()
client, err := compute.NewInstancesRESTClient(ctx)
return client, err
}

func ListInstances(client *compute.InstancesClient, projectID, zone string) *compute.InstanceIterator {
ctx := context.Background()
request := &computepb.ListInstancesRequest{
Project: projectID,
Zone: zone,
}
return client.List(ctx, request)
}

0 comments on commit 5078885

Please sign in to comment.