Merge pull request #106 from mailgun/thrawn/develop

Added optional os and golang internal metrics collectors

CHANGELOG

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0-rc.7] - 2021-08-20
+## Added
+* Added optional os and golang internal metrics collectors
+
 ## [2.0.0-rc.6] - 2021-08-20
 ## Changes
 * JSON responses are now back to their original camel_case form

Dockerfile

@@ -1,5 +1,5 @@
 # Build image
-FROM golang:1.15.4 as build
+FROM golang:1.17 as build
 
 WORKDIR /go/src
 

config.go

@@ -39,6 +39,7 @@ import (
 	"google.golang.org/grpc"
 )
 
+// BehaviorConfig controls the handling of rate limits in the cluster
 type BehaviorConfig struct {
 	// How long we should wait for a batched response from a peer
 	BatchTimeout time.Duration
@@ -62,7 +63,7 @@ type BehaviorConfig struct {
 	MultiRegionBatchLimit int
 }
 
-// config for a gubernator instance
+// Config for a gubernator instance
 type Config struct {
 	// (Required) A list of GRPC servers to register our instance with
 	GRPCServers []*grpc.Server
@@ -143,7 +144,7 @@ type PeerInfo struct {
 	IsOwner bool `json:"is-owner,omitempty"`
 }
 
-// Returns the hash key used to identify this peer in the Picker.
+// HashKey returns the hash key used to identify this peer in the Picker.
 func (p PeerInfo) HashKey() string {
 	return p.GRPCAddress
 }
@@ -199,6 +200,9 @@ type DaemonConfig struct {
 	// (Optional) TLS Configuration; SpawnDaemon() will modify the passed TLS config in an
 	// attempt to build a complete TLS config if one is not provided.
 	TLS *TLSConfig
+
+	// (Optional) Metrics Flags which enable or disable collection of some types of metrics
+	MetricFlags MetricFlags
 }
 
 func (d *DaemonConfig) ClientTLS() *tls.Config {
@@ -241,6 +245,7 @@ func SetupDaemonConfig(logger *logrus.Logger, configFile string) (DaemonConfig,
 	setter.SetDefault(&conf.CacheSize, getEnvInteger(log, "GUBER_CACHE_SIZE"), 50_000)
 	setter.SetDefault(&conf.AdvertiseAddress, os.Getenv("GUBER_ADVERTISE_ADDRESS"), conf.GRPCListenAddress)
 	setter.SetDefault(&conf.DataCenter, os.Getenv("GUBER_DATA_CENTER"), "")
+	setter.SetDefault(&conf.MetricFlags, getEnvMetricFlags(log, "GUBER_METRIC_FLAGS"))
 
 	advAddr, advPort, err := net.SplitHostPort(conf.AdvertiseAddress)
 	if err != nil {
@@ -369,10 +374,6 @@ func SetupDaemonConfig(logger *logrus.Logger, configFile string) (DaemonConfig,
 		}
 	}
 
-	if anyHasPrefix("GUBER_ETCD_", os.Environ()) {
-		log.Debug("ETCD peer pool config found")
-	}
-
 	// If env contains any TLS configuration
 	if anyHasPrefix("GUBER_ETCD_TLS_", os.Environ()) {
 		if err := setupEtcdTLS(conf.EtcdPoolConf.EtcdConfig); err != nil {

daemon.go

@@ -30,6 +30,7 @@ import (
 	"github.com/mailgun/holster/v4/syncutil"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -221,6 +222,20 @@ func (s *Daemon) Start(ctx context.Context) error {
 	// Serve the JSON Gateway and metrics handlers via standard HTTP/1
 	mux := http.NewServeMux()
 
+	// Optionally collect process metrics
+	if s.conf.MetricFlags.Has(FlagOSMetrics) {
+		s.log.Debug("Collecting OS Metrics")
+		s.promRegister.MustRegister(collectors.NewProcessCollector(
+			collectors.ProcessCollectorOpts{Namespace: "gubernator"},
+		))
+	}
+
+	// Optionally collect golang internal metrics
+	if s.conf.MetricFlags.Has(FlagGolangMetrics) {
+		s.log.Debug("Collecting Golang Metrics")
+		s.promRegister.MustRegister(collectors.NewGoCollector())
+	}
+
 	mux.Handle("/metrics", promhttp.InstrumentMetricHandler(
 		s.promRegister, promhttp.HandlerFor(s.promRegister, promhttp.HandlerOpts{}),
 	))

docker-compose.yaml

@@ -65,6 +65,7 @@ services:
       # A Comma separated list of known gubernator nodes
       - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2
       #- GUBER_DATA_CENTER=us-west-2
+      #- GUBER_METRIC_FLAGS=golang,os
     ports:
       - "9381:81"
       - "9380:80"

example.conf

@@ -29,6 +29,13 @@ GUBER_ADVERTISE_ADDRESS=localhost:9990
 # If value is zero (default) time is infinity
 # GUBER_GRPC_MAX_CONN_AGE_SEC=30
 
+# A list of optional prometheus metric collection
+# os - collect process metrics
+#      See https://pkg.go.dev/github.com/prometheus/[email protected]/prometheus/collectors#NewProcessCollector
+# golang - collect golang internal metrics
+#          See https://pkg.go.dev/github.com/prometheus/[email protected]/prometheus/collectors#NewGoCollector
+# GUBER_METRIC_FLAGS=os,golang
+
 
 ############################
 # Behavior Config

flags.go

@@ -0,0 +1,43 @@
+package gubernator
+
+import "github.com/sirupsen/logrus"
+
+const (
+	FlagOSMetrics MetricFlags = 1 << iota
+	FlagGolangMetrics
+)
+
+type MetricFlags int64
+
+func (f *MetricFlags) Set(flag MetricFlags, set bool) {
+	if set {
+		*f = *f | flag
+	} else {
+		mask := *f ^ flag
+		*f &= mask
+	}
+}
+
+func (f *MetricFlags) Has(flag MetricFlags) bool {
+	return *f&flag != 0
+}
+
+func getEnvMetricFlags(log logrus.FieldLogger, name string) MetricFlags {
+	flags := getEnvSlice(name)
+	if len(flags) == 0 {
+		return 0
+	}
+	var result MetricFlags
+
+	for _, f := range flags {
+		switch f {
+		case "os":
+			result.Set(FlagOSMetrics, true)
+		case "golang":
+			result.Set(FlagGolangMetrics, true)
+		default:
+			log.Errorf("invalid flag '%s' for '%s' valid options are ['os', 'golang']", f, name)
+		}
+	}
+	return result
+}

functional_test.go

@@ -932,7 +932,7 @@ func TestLeakyBucketDivBug(t *testing.T) {
 	assert.Equal(t, int64(2000), resp.Responses[0].Limit)
 }
 
-func TestMutliRegion(t *testing.T) {
+func TestMultiRegion(t *testing.T) {
 
 	// TODO: Queue a rate limit with multi region behavior on the DataCenterNone cluster
 	// TODO: Check the immediate response is correct

gubernator.go

@@ -475,7 +475,7 @@ func (s *V1Instance) SetPeers(peerInfo []PeerInfo) {
 	}
 }
 
-// GetPeers returns a peer client for the hash key provided
+// GetPeer returns a peer client for the hash key provided
 func (s *V1Instance) GetPeer(key string) (*PeerClient, error) {
 	s.peerMutex.RLock()
 	peer, err := s.conf.LocalPicker.Get(key)

version

@@ -1 +1 @@
-1.0.0-rc.8
+2.0.0-rc.7