diff --git a/cmd/cluster/health.go b/cmd/cluster/health.go index 1fd329b2..0efab7d3 100644 --- a/cmd/cluster/health.go +++ b/cmd/cluster/health.go @@ -1,6 +1,7 @@ package cluster import ( + "errors" "fmt" "log" "strconv" @@ -11,6 +12,7 @@ import ( "github.com/openshift/osdctl/pkg/osdCloud" "github.com/openshift/osdctl/pkg/utils" "github.com/spf13/cobra" + "google.golang.org/api/iterator" "gopkg.in/yaml.v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" ) @@ -96,59 +98,123 @@ func (o *healthOptions) run() error { healthObject.Expected.Worker = int(cluster.Nodes().Compute()) } - awsClient, err := osdCloud.GenerateAWSClientForCluster(o.awsProfile, o.clusterID) - if err != nil { - return err - } - - instances, err := awsClient.DescribeInstances(&ec2.DescribeInstancesInput{}) - if err != nil { - return err - } runningMasters := 0 runningInfra := 0 runningWorkers := 0 totalStopped := 0 totalCluster := 0 - //Here we count the number of customer's running worker, infra and master instances in the cluster in the given region. To decide if the instance belongs to the cluster we are checking the Name Tag on the instance. - for idx := range instances.Reservations { - for _, inst := range instances.Reservations[idx].Instances { - tags := inst.Tags - for _, t := range tags { - if *t.Key == "Name" { - if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "master") { - totalCluster += 1 - if *inst.State.Name == "running" { - runningMasters += 1 - } - if *inst.State.Name == "stopped" { - totalStopped += 1 - } - - } else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "infra") { - totalCluster += 1 - if *inst.State.Name == "running" { - runningInfra += 1 - } - if *inst.State.Name == "stopped" { - totalStopped += 1 - } - } else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "worker") { - totalCluster += 1 - if *inst.State.Name == "running" { - runningWorkers += 1 - } - if *inst.State.Name == "stopped" { - totalStopped += 1 - } - + if cluster.CloudProvider().ID() == "gcp" { + clusterResources, err := ocmClient.ClustersMgmt().V1().Clusters().Cluster(o.clusterID).Resources().Live().Get().Send() + if err != nil { + return err + } + projectClaimRaw, found := clusterResources.Body().Resources()["gcp_project_claim"] + if !found { + return fmt.Errorf("The gcp_project_claim was not found in the ocm resource") + } + projectClaim, err := osdCloud.ParseGcpProjectClaim(projectClaimRaw) + if err != nil { + log.Printf("Unmarshalling GCP projectClaim failed: %v\n", err) + return err + } + projectId := projectClaim.Spec.GcpProjectID + zones := cluster.Nodes().AvailabilityZones() + if projectId == "" || len(zones) == 0 { + return fmt.Errorf("ProjectID or Zones empty - aborting") + } + gcpClient, err := osdCloud.GenerateGCPComputeInstancesClient() + defer gcpClient.Close() + if err != nil { + return err + } + ownedLabel := "kubernetes-io-cluster-" + cluster.InfraID() + for _, zone := range zones { + instances := osdCloud.ListInstances(gcpClient, projectId, zone) + for { + instance, err := instances.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + name := instance.GetName() + state := instance.GetStatus() + labels := instance.GetLabels() + belongsToCluster := false + for label := range labels { + if label == ownedLabel { + belongsToCluster = true + } + } + if !belongsToCluster { + log.Printf("Skipping a machine not belonging to the cluster: %s\n", name) + continue + } + totalCluster += 1 + if state != "RUNNING" { + totalStopped += 1 + } else { + if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "master") { + runningMasters += 1 + } else if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "infra") { + runningInfra += 1 + } else if strings.HasPrefix(name, cluster.InfraID()) && strings.Contains(name, "worker") { + runningWorkers += 1 } } } + } + } else if cluster.CloudProvider().ID() == "aws" { + awsClient, err := osdCloud.GenerateAWSClientForCluster(o.awsProfile, o.clusterID) + if err != nil { + return err + } + instances, err := awsClient.DescribeInstances(&ec2.DescribeInstancesInput{}) + if err != nil { + return err } + //Here we count the number of customer's running worker, infra and master instances in the cluster in the given region. To decide if the instance belongs to the cluster we are checking the Name Tag on the instance. + for idx := range instances.Reservations { + for _, inst := range instances.Reservations[idx].Instances { + tags := inst.Tags + for _, t := range tags { + if *t.Key == "Name" { + if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "master") { + totalCluster += 1 + if *inst.State.Name == "running" { + runningMasters += 1 + } + if *inst.State.Name == "stopped" { + totalStopped += 1 + } + + } else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "infra") { + totalCluster += 1 + if *inst.State.Name == "running" { + runningInfra += 1 + } + if *inst.State.Name == "stopped" { + totalStopped += 1 + } + } else if strings.HasPrefix(*t.Value, cluster.InfraID()) && strings.Contains(*t.Value, "worker") { + totalCluster += 1 + if *inst.State.Name == "running" { + runningWorkers += 1 + } + if *inst.State.Name == "stopped" { + totalStopped += 1 + } + } + } + } + } + } + } else { + return errors.New(fmt.Sprintf("Unknown cloud provider found: %s", cluster.CloudProvider().ID())) } healthObject.Actual.Stopped = totalStopped diff --git a/go.mod b/go.mod index 7f28e222..729f9329 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/openshift/osdctl go 1.19 require ( + cloud.google.com/go/compute v1.7.0 github.com/PagerDuty/go-pagerduty v1.5.1 github.com/andygrunwald/go-jira v1.16.0 github.com/aws/aws-sdk-go v1.44.66 @@ -29,6 +30,8 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.13.0 + google.golang.org/api v0.84.0 + google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 @@ -75,6 +78,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect @@ -83,6 +87,8 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa // indirect + github.com/googleapis/gax-go/v2 v2.4.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -133,6 +139,7 @@ require ( github.com/trivago/tgo v1.0.7 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a // indirect + go.opencensus.io v0.23.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.4.0 // indirect @@ -142,6 +149,7 @@ require ( golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 5c2d370d..d65cc96f 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,7 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -41,6 +42,7 @@ cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJW cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= @@ -330,6 +332,7 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -420,6 +423,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa h1:7MYGT2XEMam7Mtzv1yDUYXANedWvwk3HKkR3MyGowy8= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -427,6 +431,7 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -930,6 +935,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -1367,6 +1373,7 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1459,6 +1466,7 @@ google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1495,6 +1503,7 @@ google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/osdCloud/gcp.go b/pkg/osdCloud/gcp.go new file mode 100644 index 00000000..79b3aef4 --- /dev/null +++ b/pkg/osdCloud/gcp.go @@ -0,0 +1,40 @@ +package osdCloud + +import ( + "context" + "encoding/json" + + compute "cloud.google.com/go/compute/apiv1" + computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" +) + +type GcpProjectClaimSpec struct { + GcpProjectID string `json:"gcpProjectID"` +} +type GcpProjectClaim struct { + Spec GcpProjectClaimSpec `json:"spec"` +} + +func ParseGcpProjectClaim(raw string) (*GcpProjectClaim, error) { + var projectClaim GcpProjectClaim + err := json.Unmarshal([]byte(raw), &projectClaim) + if err != nil { + return nil, err + } + return &projectClaim, nil +} + +func GenerateGCPComputeInstancesClient() (*compute.InstancesClient, error) { + ctx := context.Background() + client, err := compute.NewInstancesRESTClient(ctx) + return client, err +} + +func ListInstances(client *compute.InstancesClient, projectID, zone string) *compute.InstanceIterator { + ctx := context.Background() + request := &computepb.ListInstancesRequest{ + Project: projectID, + Zone: zone, + } + return client.List(ctx, request) +}