Bug fixes and code improvements

CHANGELOG

@@ -11,8 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Renamed functions to ensure clarity of version
 * Removed deprecated `EtcdAdvertiseAddress` config option
 * Refactored configuration options
-* 'member-list' metadata no longer assumes the member-list address is the same as
-  the gubernator advertise address.
+* 'member-list' metadata no longer assumes the member-list address is the same
+  as the gubernator advertise address.
+* Now MD5 sums the peer address key when using replicated hash. This ensures
+  better key distribution when using domain names or ip address that are very
+  similar. (gubernator-1, gubernator-2, etc...)
+* Now defaults to `replicated-hash` if `GUBER_PEER_PICKER` is unset
+* Added support for DataCenter fields when using etcd discovery
 
 ## [0.9.2] - 2020-10-23
 ### Change

Dockerfile

@@ -10,11 +10,12 @@ RUN go mod download
 
 # Copy the local package files to the container
 ADD . /go/src
-ENV VERSION=dev-build
 
-# Build the bot inside the container
+ARG VERSION
+
+# Build the server inside the container
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo \
-    -ldflags "-w -s -X main.Version=${VERSION}" -o /gubernator /go/src/cmd/gubernator/main.go /go/src/cmd/gubernator/config.go
+    -ldflags "-w -s -X main.Version=$VERSION" -o /gubernator /go/src/cmd/gubernator/main.go
 
 # Create our deploy image
 FROM scratch

cmd/gubernator/main.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -48,6 +49,7 @@ var Version = "dev-build"
 func main() {
 	var configFile string
 
+	logrus.Infof("Gubernator %s (%s/%s)", Version, runtime.GOARCH, runtime.GOOS)
 	flags := flag.NewFlagSet("gubernator", flag.ContinueOnError)
 	flags.StringVar(&configFile, "config", "", "yaml config file")
 	flags.BoolVar(&gubernator.DebugEnabled, "debug", false, "enable debug")
@@ -85,10 +87,10 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 	klog.InitFlags(nil)
 	flag.Set("logtostderr", "true")
 
-	if gubernator.DebugEnabled || os.Getenv("GUBER_DEBUG") != "" {
+	setter.SetDefault(&gubernator.DebugEnabled, getEnvBool("GUBER_DEBUG"))
+	if gubernator.DebugEnabled {
 		logrus.SetLevel(logrus.DebugLevel)
 		logrus.Debug("Debug enabled")
-		gubernator.DebugEnabled = true
 	}
 
 	if configFile != "" {
@@ -103,8 +105,19 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 	setter.SetDefault(&conf.HTTPListenAddress, os.Getenv("GUBER_HTTP_ADDRESS"), "localhost:80")
 	setter.SetDefault(&conf.CacheSize, getEnvInteger("GUBER_CACHE_SIZE"), 50_000)
 	setter.SetDefault(&conf.DataCenter, os.Getenv("GUBER_DATA_CENTER"), "")
+
 	setter.SetDefault(&conf.AdvertiseAddress, os.Getenv("GUBER_ADVERTISE_ADDRESS"), conf.GRPCListenAddress)
 
+	advAddr, advPort, err := net.SplitHostPort(conf.AdvertiseAddress)
+	if err != nil {
+		return conf, errors.Wrap(err, "GUBER_ADVERTISE_ADDRESS is invalid; expected format is `address:port`")
+	}
+	advAddr, err = gubernator.ResolveHostIP(advAddr)
+	if err != nil {
+		return conf, errors.Wrap(err, "failed to discover host ip for GUBER_ADVERTISE_ADDRESS")
+	}
+	conf.AdvertiseAddress = net.JoinHostPort(advAddr, advPort)
+
 	// Behaviors
 	setter.SetDefault(&conf.Behaviors.BatchTimeout, getEnvDuration("GUBER_BATCH_TIMEOUT"))
 	setter.SetDefault(&conf.Behaviors.BatchLimit, getEnvInteger("GUBER_BATCH_LIMIT"))
@@ -119,7 +132,7 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 	setter.SetDefault(&conf.Behaviors.MultiRegionSyncWait, getEnvDuration("GUBER_MULTI_REGION_SYNC_WAIT"))
 
 	choices := []string{"member-list", "k8s", "etcd"}
-	setter.SetDefault(&conf.PeerDiscoveryType, getEnvDuration("GUBER_PEER_DISCOVERY_TYPE"), "member-list")
+	setter.SetDefault(&conf.PeerDiscoveryType, os.Getenv("GUBER_PEER_DISCOVERY_TYPE"), "member-list")
 	if !slice.ContainsString(conf.PeerDiscoveryType, choices, nil) {
 		return conf, fmt.Errorf("GUBER_PEER_DISCOVERY_TYPE is invalid; choices are [%s]`", strings.Join(choices, ","))
 	}
@@ -132,15 +145,10 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 	setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Username, os.Getenv("GUBER_ETCD_USER"))
 	setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Password, os.Getenv("GUBER_ETCD_PASSWORD"))
 	setter.SetDefault(&conf.EtcdPoolConf.AdvertiseAddress, os.Getenv("GUBER_ETCD_ADVERTISE_ADDRESS"), conf.AdvertiseAddress)
-
-	// Memberlist Config
-	addr, _, err := net.SplitHostPort(conf.GRPCListenAddress)
-	if err != nil {
-		return conf, errors.Wrap(err, "GUBER_GRPC_ADDRESS is invalid; expected format is `address:port`")
-	}
+	setter.SetDefault(&conf.EtcdPoolConf.DataCenter, os.Getenv("GUBER_ETCD_DATA_CENTER"), conf.DataCenter)
 
 	setter.SetDefault(&conf.MemberListPoolConf.AdvertiseAddress, os.Getenv("GUBER_MEMBERLIST_ADVERTISE_ADDRESS"), conf.AdvertiseAddress)
-	setter.SetDefault(&conf.MemberListPoolConf.MemberListAddress, os.Getenv("GUBER_MEMBERLIST_ADDRESS"), fmt.Sprintf("%s:7946", addr))
+	setter.SetDefault(&conf.MemberListPoolConf.MemberListAddress, os.Getenv("GUBER_MEMBERLIST_ADDRESS"), fmt.Sprintf("%s:7946", advAddr))
 	setter.SetDefault(&conf.MemberListPoolConf.KnownNodes, getEnvSlice("GUBER_MEMBERLIST_KNOWN_NODES"), []string{})
 	setter.SetDefault(&conf.MemberListPoolConf.DataCenter, conf.DataCenter)
 
@@ -168,11 +176,11 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 				return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]",
 					hash, validHashKeys(hashFuncs))
 			}
-			conf.Picker = gubernator.NewConsistantHash(fn)
+			conf.Picker = gubernator.NewConsistentHash(fn)
 
 		case "replicated-hash":
-			setter.SetDefault(&replicas, getEnvInteger("GUBER_REPLICATED_HASH_REPLICAS"), 1)
-			conf.Picker = gubernator.NewReplicatedConsistantHash(nil, replicas)
+			setter.SetDefault(&replicas, getEnvInteger("GUBER_REPLICATED_HASH_REPLICAS"), gubernator.DefaultReplicas)
+			conf.Picker = gubernator.NewReplicatedConsistentHash(nil, replicas)
 			setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a")
 			hashFuncs := map[string]gubernator.HashFunc64{
 				"fnv1a": fnv1a.HashBytes64,
@@ -183,7 +191,7 @@ func confFromFile(configFile string) (gubernator.DaemonConfig, error) {
 				return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]",
 					hash, validHash64Keys(hashFuncs))
 			}
-			conf.Picker = gubernator.NewReplicatedConsistantHash(fn, replicas)
+			conf.Picker = gubernator.NewReplicatedConsistentHash(fn, replicas)
 		default:
 			return conf, errors.Errorf("'GUBER_PEER_PICKER=%s' is invalid; choices are ['replicated-hash', 'consistent-hash']", pp)
 		}
@@ -284,6 +292,19 @@ func anyHasPrefix(prefix string, items []string) bool {
 	return false
 }
 
+func getEnvBool(name string) bool {
+	v := os.Getenv(name)
+	if v == "" {
+		return false
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		log.WithError(err).Errorf("while parsing '%s' as an boolean", name)
+		return false
+	}
+	return b
+}
+
 func getEnvInteger(name string) int {
 	v := os.Getenv(name)
 	if v == "" {

config.go

@@ -86,6 +86,24 @@ type BehaviorConfig struct {
 	MultiRegionBatchLimit int
 }
 
+type PeerInfo struct {
+	// (Optional) The name of the data center this peer is in. Leave blank if not using multi data center support.
+	DataCenter string
+	// (Optional) The http address:port of the peer
+	HTTPAddress string
+	// (Required) The grpc address:port of the peer
+	GRPCAddress string
+	// (Optional) Is true if PeerInfo is for this instance of gubernator
+	IsOwner bool
+}
+
+// Returns the hash key used to identify this peer in the Picker.
+func (p PeerInfo) HashKey() string {
+	return p.GRPCAddress
+}
+
+type UpdateFunc func([]PeerInfo)
+
 func (c *Config) SetDefaults() error {
 	setter.SetDefault(&c.Behaviors.BatchTimeout, time.Millisecond*500)
 	setter.SetDefault(&c.Behaviors.BatchLimit, maxBatchSize)
@@ -99,7 +117,7 @@ func (c *Config) SetDefaults() error {
 	setter.SetDefault(&c.Behaviors.MultiRegionBatchLimit, maxBatchSize)
 	setter.SetDefault(&c.Behaviors.MultiRegionSyncWait, time.Second)
 
-	setter.SetDefault(&c.LocalPicker, NewConsistantHash(nil))
+	setter.SetDefault(&c.LocalPicker, NewReplicatedConsistentHash(nil, DefaultReplicas))
 	setter.SetDefault(&c.RegionPicker, NewRegionPicker(nil))
 	setter.SetDefault(&c.Cache, NewLRUCache(0))
 

docker-compose-etcd.yaml

@@ -0,0 +1,101 @@
+version: '3'
+services:
+  etcd:
+    image: quay.io/coreos/etcd:v3.3.10
+    command: >
+      /usr/local/bin/etcd
+      -name etcd0
+      -advertise-client-urls http://localhost:2379
+      -listen-client-urls http://0.0.0.0:2379
+      -initial-advertise-peer-urls http://0.0.0.0:2380
+      -listen-peer-urls http://0.0.0.0:2380
+      -initial-cluster-token etcd-cluster-1
+      -initial-cluster etcd0=http://0.0.0.0:2380
+      -initial-cluster-state new
+    ports:
+      - "2379:2379"
+
+  gubernator-1:
+    image: thrawn01/gubernator:latest
+    command: "/gubernator"
+    environment:
+      # The address GRPC requests will listen on
+      - GUBER_GRPC_ADDRESS=0.0.0.0:81
+      # The address HTTP requests will listen on
+      - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # Choose the etcd peer discovery type
+      - GUBER_PEER_DISCOVERY_TYPE=etcd
+      # A comma separated list of etcd nodes
+      - GUBER_ETCD_ENDPOINTS=etcd:2379
+      # The key prefix used in the etcd store
+      - GUBER_ETCD_KEY_PREFIX=/gubernator-docker
+      # The address that is advertised to other peers
+      - GUBER_ETCD_ADVERTISE_ADDRESS=gubernator-1:81
+      #- GUBER_DATA_CENTER=us-east-1
+    ports:
+      - "9081:81"
+      - "9080:80"
+
+  gubernator-2:
+    image: thrawn01/gubernator:latest
+    command: "/gubernator"
+    environment:
+      # The address GRPC requests will listen on
+      - GUBER_GRPC_ADDRESS=0.0.0.0:81
+      # The address HTTP requests will listen on
+      - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # Choose the etcd peer discovery type
+      - GUBER_PEER_DISCOVERY_TYPE=etcd
+      # A comma separated list of etcd nodes
+      - GUBER_ETCD_ENDPOINTS=etcd:2379
+      # The key prefix used in the etcd store
+      - GUBER_ETCD_KEY_PREFIX=/gubernator-docker
+      # The address that is advertised to other peers
+      - GUBER_ETCD_ADVERTISE_ADDRESS=gubernator-2:81
+      #- GUBER_DATA_CENTER=us-east-1
+    ports:
+      - "9181:81"
+      - "9180:80"
+
+  gubernator-3:
+    image: thrawn01/gubernator:latest
+    command: "/gubernator"
+    environment:
+      # The address GRPC requests will listen on
+      - GUBER_GRPC_ADDRESS=0.0.0.0:81
+      # The address HTTP requests will listen on
+      - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # Choose the etcd peer discovery type
+      - GUBER_PEER_DISCOVERY_TYPE=etcd
+      # A comma separated list of etcd nodes
+      - GUBER_ETCD_ENDPOINTS=etcd:2379
+      # The key prefix used in the etcd store
+      - GUBER_ETCD_KEY_PREFIX=/gubernator-docker
+      # The address that is advertised to other peers
+      - GUBER_ETCD_ADVERTISE_ADDRESS=gubernator-3:81
+      #- GUBER_DATA_CENTER=us-west-2
+    ports:
+      - "9281:81"
+      - "9280:80"
+
+  gubernator-4:
+    image: thrawn01/gubernator:latest
+    command: "/gubernator"
+    environment:
+      - GUBER_DEBUG=true
+      # The address GRPC requests will listen on
+      - GUBER_GRPC_ADDRESS=0.0.0.0:81
+      # The address HTTP requests will listen on
+      - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # Choose the etcd peer discovery type
+      - GUBER_PEER_DISCOVERY_TYPE=etcd
+      # A comma separated list of etcd nodes
+      - GUBER_ETCD_ENDPOINTS=etcd:2379
+      # The key prefix used in the etcd store
+      - GUBER_ETCD_KEY_PREFIX=/gubernator-docker
+      # The address that is advertised to other peers
+      - GUBER_ADVERTISE_ADDRESS=gubernator-4:81
+      #- GUBER_DATA_CENTER=us-west-2
+    ports:
+      - "9381:81"
+      - "9380:80"

docker-compose.yaml

@@ -8,14 +8,16 @@ services:
       - GUBER_GRPC_ADDRESS=0.0.0.0:81
       # The address HTTP requests will listen on
       - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # The address that is advertised to other peers
+      - GUBER_ADVERTISE_ADDRESS=gubernator-1:81
       # Max size of the cache; The cache size will never grow beyond this size.
       - GUBER_CACHE_SIZE=50000
       # A comma separated list of known gubernator nodes
-      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2,gubernator-3,gubernator-4
-      - GUBER_DATA_CENTER=us-east-1
+      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1
+      #- GUBER_DATA_CENTER=us-east-1
     ports:
-      - "8081:81"
-      - "8080:80"
+      - "9081:81"
+      - "9080:80"
 
   gubernator-2:
     image: thrawn01/gubernator:latest
@@ -25,14 +27,16 @@ services:
       - GUBER_GRPC_ADDRESS=0.0.0.0:81
       # The address HTTP requests will listen on
       - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # The address that is advertised to other peers
+      - GUBER_ADVERTISE_ADDRESS=gubernator-2:81
       # Max size of the cache; The cache size will never grow beyond this size.
       - GUBER_CACHE_SIZE=50000
       # A comma separated list of known gubernator nodes
-      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2,gubernator-3,gubernator-4
-      - GUBER_DATA_CENTER=us-east-1
+      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1
+      #- GUBER_DATA_CENTER=us-east-1
     ports:
-      - "8181:81"
-      - "8180:80"
+      - "9181:81"
+      - "9180:80"
 
   gubernator-3:
     image: thrawn01/gubernator:latest
@@ -42,28 +46,33 @@ services:
       - GUBER_GRPC_ADDRESS=0.0.0.0:81
       # The address HTTP requests will listen on
       - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # The address that is advertised to other peers
+      - GUBER_ADVERTISE_ADDRESS=gubernator-3:81
       # Max size of the cache; The cache size will never grow beyond this size.
       - GUBER_CACHE_SIZE=50000
       # A comma separated list of known gubernator nodes
-      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2,gubernator-3,gubernator-4
-      - GUBER_DATA_CENTER=us-west-2
+      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1
+      #- GUBER_DATA_CENTER=us-west-2
     ports:
-      - "8281:81"
-      - "8280:80"
+      - "9281:81"
+      - "9280:80"
 
   gubernator-4:
     image: thrawn01/gubernator:latest
     command: "/gubernator"
     environment:
+      - GUBER_DEBUG=true
       # The address GRPC requests will listen on
       - GUBER_GRPC_ADDRESS=0.0.0.0:81
       # The address HTTP requests will listen on
       - GUBER_HTTP_ADDRESS=0.0.0.0:80
+      # The address that is advertised to other peers
+      - GUBER_ADVERTISE_ADDRESS=gubernator-4:81
       # Max size of the cache; The cache size will never grow beyond this size.
       - GUBER_CACHE_SIZE=50000
-      # A Comma separate list of known gubernator nodes
-      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2,gubernator-3,gubernator-4
-      - GUBER_DATA_CENTER=us-west-2
+      # A Comma separated list of known gubernator nodes
+      - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1
+      #- GUBER_DATA_CENTER=us-west-2
     ports:
-      - "8381:81"
-      - "8380:80"
+      - "9381:81"
+      - "9380:80"