diff --git a/Dockerfile b/Dockerfile index d430f64b..6b7f8795 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build image -FROM golang:1.15.1 as build +FROM golang:1.15.4 as build WORKDIR /go/src diff --git a/README.md b/README.md index f50507d5..fb34c09c 100644 --- a/README.md +++ b/README.md @@ -252,15 +252,15 @@ don't have either, the docker-compose method is the simplest way to try gubernat ##### Docker with existing etcd cluster ```bash -$ docker run -p 8081:81 -p 8080:80 -e GUBER_ETCD_ENDPOINTS=etcd1:2379,etcd2:2379 \ +$ docker run -p 8081:81 -p 9080:80 -e GUBER_ETCD_ENDPOINTS=etcd1:2379,etcd2:2379 \ thrawn01/gubernator:latest -# Hit the API at localhost:8080 (GRPC is at 8081) -$ curl http://localhost:8080/v1/HealthCheck +# Hit the API at localhost:9080 +$ curl http://localhost:9080/v1/HealthCheck ``` ##### Docker compose -The docker compose file includes a local etcd server and 2 gubernator instances +The docker compose file uses member-list for peer discovery ```bash # Download the docker-compose file $ curl -O https://raw.githubusercontent.com/mailgun/gubernator/master/docker-compose.yaml @@ -271,8 +271,8 @@ $ vi docker-compose.yaml # Run the docker container $ docker-compose up -d -# Hit the API at localhost:8080 (GRPC is at 8081) -$ curl http://localhost:8080/v1/HealthCheck +# Hit the API at localhost:9080 (GRPC is at 9081) +$ curl http://localhost:9080/v1/HealthCheck ``` ##### Kubernetes diff --git a/cli-tls.conf b/cli-tls.conf new file mode 100644 index 00000000..2394c482 --- /dev/null +++ b/cli-tls.conf @@ -0,0 +1,5 @@ +GUBER_DEBUG=true +GUBER_GRPC_ADDRESS=localhost:9081 +GUBER_TLS_CA=certs/ca.pem +GUBER_TLS_KEY=certs/gubernator.key +GUBER_TLS_CERT=certs/gubernator.pem diff --git a/cmd/gubernator-cli/main.go b/cmd/gubernator-cli/main.go index 4b5f9844..9681d315 100644 --- a/cmd/gubernator-cli/main.go +++ b/cmd/gubernator-cli/main.go @@ -18,6 +18,8 @@ package main import ( "context" + "errors" + "flag" "fmt" "math/rand" "os" @@ -25,12 +27,16 @@ import ( "github.com/davecgh/go-spew/spew" guber "github.com/mailgun/gubernator" "github.com/mailgun/holster/v3/clock" + "github.com/mailgun/holster/v3/setter" "github.com/mailgun/holster/v3/syncutil" + "github.com/sirupsen/logrus" ) +var log *logrus.Logger + func checkErr(err error) { if err != nil { - fmt.Fprintf(os.Stderr, "error: %s\n", err) + log.Errorf(err.Error()) os.Exit(1) } } @@ -40,12 +46,29 @@ func randInt(min, max int) int64 { } func main() { - if len(os.Args) < 2 { - fmt.Printf("Please provide an gubernator GRPC endpoint address\n") - os.Exit(1) + var configFile, GRPCAddress string + var err error + + log = logrus.StandardLogger() + flags := flag.NewFlagSet("gubernator", flag.ContinueOnError) + flags.StringVar(&configFile, "config", "", "environment config file") + flags.StringVar(&GRPCAddress, "e", "", "the gubernator GRPC endpoint address") + checkErr(flags.Parse(os.Args[1:])) + + conf, err := guber.SetupDaemonConfig(log, configFile) + checkErr(err) + setter.SetOverride(&conf.GRPCListenAddress, GRPCAddress) + + if configFile == "" && GRPCAddress == "" && os.Getenv("GUBER_GRPC_ADDRESS") == "" { + checkErr(errors.New("please provide a GRPC endpoint via -e or from a config " + + "file via -config or set the env GUBER_GRPC_ADDRESS")) } - client, err := guber.DialV1Server(os.Args[1], nil) + err = guber.SetupTLS(conf.TLS) + checkErr(err) + + log.Infof("Connecting to '%s'...\n", conf.GRPCListenAddress) + client, err := guber.DialV1Server(conf.GRPCListenAddress, conf.ClientTLS()) checkErr(err) // Generate a selection of rate limits with random limits diff --git a/cmd/gubernator/main.go b/cmd/gubernator/main.go index 88e22931..c1832cd2 100644 --- a/cmd/gubernator/main.go +++ b/cmd/gubernator/main.go @@ -18,45 +18,39 @@ package main import ( "context" - "crypto/tls" - "crypto/x509" "flag" - "fmt" - "io/ioutil" - "net" "os" "os/signal" "runtime" - "strconv" - "strings" - etcd "github.com/coreos/etcd/clientv3" - "github.com/davecgh/go-spew/spew" "github.com/mailgun/gubernator" "github.com/mailgun/holster/v3/clock" - "github.com/mailgun/holster/v3/setter" - "github.com/mailgun/holster/v3/slice" - "github.com/pkg/errors" - "github.com/segmentio/fasthash/fnv1" - "github.com/segmentio/fasthash/fnv1a" "github.com/sirupsen/logrus" "k8s.io/klog" ) -var log = logrus.WithField("category", "server") +var log = logrus.WithField("category", "gubernator") var Version = "dev-build" func main() { var configFile string + var err error logrus.Infof("Gubernator %s (%s/%s)", Version, runtime.GOARCH, runtime.GOOS) flags := flag.NewFlagSet("gubernator", flag.ContinueOnError) - flags.StringVar(&configFile, "config", "", "yaml config file") + flags.StringVar(&configFile, "config", "", "environment config file") flags.BoolVar(&gubernator.DebugEnabled, "debug", false, "enable debug") checkErr(flags.Parse(os.Args[1:]), "while parsing flags") + // in order to prevent logging to /tmp by k8s.io/client-go + // and other kubernetes related dependencies which are using + // klog (https://github.com/kubernetes/klog), we need to + // initialize klog in the way it prints to stderr only. + klog.InitFlags(nil) + flag.Set("logtostderr", "true") + // Read our config from the environment or optional environment config file - conf, err := confFromFile(configFile) + conf, err := gubernator.SetupDaemonConfig(logrus.StandardLogger(), configFile) checkErr(err, "while getting config") ctx, cancel := context.WithTimeout(context.Background(), clock.Second*10) @@ -78,348 +72,9 @@ func main() { } } -func confFromFile(configFile string) (gubernator.DaemonConfig, error) { - var conf gubernator.DaemonConfig - - // in order to prevent logging to /tmp by k8s.io/client-go - // and other kubernetes related dependencies which are using - // klog (https://github.com/kubernetes/klog), we need to - // initialize klog in the way it prints to stderr only. - klog.InitFlags(nil) - flag.Set("logtostderr", "true") - - setter.SetDefault(&gubernator.DebugEnabled, getEnvBool("GUBER_DEBUG")) - if gubernator.DebugEnabled { - logrus.SetLevel(logrus.DebugLevel) - logrus.Debug("Debug enabled") - } - - if configFile != "" { - log.Infof("Loading env config: %s", configFile) - if err := fromEnvFile(configFile); err != nil { - return conf, err - } - } - - // Main config - setter.SetDefault(&conf.GRPCListenAddress, os.Getenv("GUBER_GRPC_ADDRESS"), "localhost:81") - setter.SetDefault(&conf.HTTPListenAddress, os.Getenv("GUBER_HTTP_ADDRESS"), "localhost:80") - setter.SetDefault(&conf.CacheSize, getEnvInteger("GUBER_CACHE_SIZE"), 50_000) - setter.SetDefault(&conf.DataCenter, os.Getenv("GUBER_DATA_CENTER"), "") - - setter.SetDefault(&conf.AdvertiseAddress, os.Getenv("GUBER_ADVERTISE_ADDRESS"), conf.GRPCListenAddress) - - advAddr, advPort, err := net.SplitHostPort(conf.AdvertiseAddress) - if err != nil { - return conf, errors.Wrap(err, "GUBER_ADVERTISE_ADDRESS is invalid; expected format is `address:port`") - } - advAddr, err = gubernator.ResolveHostIP(advAddr) - if err != nil { - return conf, errors.Wrap(err, "failed to discover host ip for GUBER_ADVERTISE_ADDRESS") - } - conf.AdvertiseAddress = net.JoinHostPort(advAddr, advPort) - - // Behaviors - setter.SetDefault(&conf.Behaviors.BatchTimeout, getEnvDuration("GUBER_BATCH_TIMEOUT")) - setter.SetDefault(&conf.Behaviors.BatchLimit, getEnvInteger("GUBER_BATCH_LIMIT")) - setter.SetDefault(&conf.Behaviors.BatchWait, getEnvDuration("GUBER_BATCH_WAIT")) - - setter.SetDefault(&conf.Behaviors.GlobalTimeout, getEnvDuration("GUBER_GLOBAL_TIMEOUT")) - setter.SetDefault(&conf.Behaviors.GlobalBatchLimit, getEnvInteger("GUBER_GLOBAL_BATCH_LIMIT")) - setter.SetDefault(&conf.Behaviors.GlobalSyncWait, getEnvDuration("GUBER_GLOBAL_SYNC_WAIT")) - - setter.SetDefault(&conf.Behaviors.MultiRegionTimeout, getEnvDuration("GUBER_MULTI_REGION_TIMEOUT")) - setter.SetDefault(&conf.Behaviors.MultiRegionBatchLimit, getEnvInteger("GUBER_MULTI_REGION_BATCH_LIMIT")) - setter.SetDefault(&conf.Behaviors.MultiRegionSyncWait, getEnvDuration("GUBER_MULTI_REGION_SYNC_WAIT")) - - choices := []string{"member-list", "k8s", "etcd"} - setter.SetDefault(&conf.PeerDiscoveryType, os.Getenv("GUBER_PEER_DISCOVERY_TYPE"), "member-list") - if !slice.ContainsString(conf.PeerDiscoveryType, choices, nil) { - return conf, fmt.Errorf("GUBER_PEER_DISCOVERY_TYPE is invalid; choices are [%s]`", strings.Join(choices, ",")) - } - - // TLS Config - setter.SetDefault(&conf.TLS, &gubernator.TLSConfig{}) - setter.SetDefault(&conf.TLS.CaFile, os.Getenv("GUBER_TLS_CA")) - setter.SetDefault(&conf.TLS.CaKeyFile, os.Getenv("GUBER_TLS_CA_KEY")) - setter.SetDefault(&conf.TLS.KeyFile, os.Getenv("GUBER_TLS_KEY")) - setter.SetDefault(&conf.TLS.CertFile, os.Getenv("GUBER_TLS_CERT")) - setter.SetDefault(&conf.TLS.AutoTLS, getEnvBool("GUBER_TLS_AUTO")) - - clientAuth := os.Getenv("GUBER_TLS_CLIENT_AUTH") - if clientAuth != "" { - clientAuthTypes := map[string]tls.ClientAuthType{ - "request-cert": tls.RequestClientCert, - "verify-cert": tls.VerifyClientCertIfGiven, - "require-any-cert": tls.RequireAnyClientCert, - "require-and-verify": tls.RequireAndVerifyClientCert, - } - t, ok := clientAuthTypes[clientAuth] - if !ok { - return conf, errors.Errorf("'GUBER_TLS_CLIENT_AUTH=%s' is invalid; choices are [%s]", - clientAuth, validClientAuthTypes(clientAuthTypes)) - } - conf.TLS.ClientAuth = t - } - setter.SetDefault(&conf.TLS.ClientAuthKeyFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_KEY")) - setter.SetDefault(&conf.TLS.ClientAuthCertFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_CERT")) - setter.SetDefault(&conf.TLS.ClientAuthCaFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_CA_CERT")) - setter.SetDefault(&conf.TLS.InsecureSkipVerify, getEnvBool("GUBER_TLS_INSECURE_SKIP_VERIFY")) - - // ETCD Config - setter.SetDefault(&conf.EtcdPoolConf.KeyPrefix, os.Getenv("GUBER_ETCD_KEY_PREFIX"), "/gubernator-peers") - setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig, &etcd.Config{}) - setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Endpoints, getEnvSlice("GUBER_ETCD_ENDPOINTS"), []string{"localhost:2379"}) - setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.DialTimeout, getEnvDuration("GUBER_ETCD_DIAL_TIMEOUT"), clock.Second*5) - setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Username, os.Getenv("GUBER_ETCD_USER")) - setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Password, os.Getenv("GUBER_ETCD_PASSWORD")) - setter.SetDefault(&conf.EtcdPoolConf.AdvertiseAddress, os.Getenv("GUBER_ETCD_ADVERTISE_ADDRESS"), conf.AdvertiseAddress) - setter.SetDefault(&conf.EtcdPoolConf.DataCenter, os.Getenv("GUBER_ETCD_DATA_CENTER"), conf.DataCenter) - - setter.SetDefault(&conf.MemberListPoolConf.AdvertiseAddress, os.Getenv("GUBER_MEMBERLIST_ADVERTISE_ADDRESS"), conf.AdvertiseAddress) - setter.SetDefault(&conf.MemberListPoolConf.MemberListAddress, os.Getenv("GUBER_MEMBERLIST_ADDRESS"), fmt.Sprintf("%s:7946", advAddr)) - setter.SetDefault(&conf.MemberListPoolConf.KnownNodes, getEnvSlice("GUBER_MEMBERLIST_KNOWN_NODES"), []string{}) - setter.SetDefault(&conf.MemberListPoolConf.DataCenter, conf.DataCenter) - - // Kubernetes Config - setter.SetDefault(&conf.K8PoolConf.Namespace, os.Getenv("GUBER_K8S_NAMESPACE"), "default") - conf.K8PoolConf.PodIP = os.Getenv("GUBER_K8S_POD_IP") - conf.K8PoolConf.PodPort = os.Getenv("GUBER_K8S_POD_PORT") - conf.K8PoolConf.Selector = os.Getenv("GUBER_K8S_ENDPOINTS_SELECTOR") - - // PeerPicker Config - if pp := os.Getenv("GUBER_PEER_PICKER"); pp != "" { - var replicas int - var hash string - - switch pp { - case "consistent-hash": - setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a") - hashFuncs := map[string]gubernator.HashFunc{ - "fnv1a": fnv1a.HashBytes32, - "fnv1": fnv1.HashBytes32, - "crc32": nil, - } - fn, ok := hashFuncs[hash] - if !ok { - return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]", - hash, validHashKeys(hashFuncs)) - } - conf.Picker = gubernator.NewConsistentHash(fn) - - case "replicated-hash": - setter.SetDefault(&replicas, getEnvInteger("GUBER_REPLICATED_HASH_REPLICAS"), gubernator.DefaultReplicas) - conf.Picker = gubernator.NewReplicatedConsistentHash(nil, replicas) - setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a") - hashFuncs := map[string]gubernator.HashFunc64{ - "fnv1a": fnv1a.HashBytes64, - "fnv1": fnv1.HashBytes64, - } - fn, ok := hashFuncs[hash] - if !ok { - return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]", - hash, validHash64Keys(hashFuncs)) - } - conf.Picker = gubernator.NewReplicatedConsistentHash(fn, replicas) - default: - return conf, errors.Errorf("'GUBER_PEER_PICKER=%s' is invalid; choices are ['replicated-hash', 'consistent-hash']", pp) - } - } - - if anyHasPrefix("GUBER_K8S_", os.Environ()) { - logrus.Debug("K8s peer pool config found") - if conf.K8PoolConf.Selector == "" { - return conf, errors.New("when using k8s for peer discovery, you MUST provide a " + - "`GUBER_K8S_ENDPOINTS_SELECTOR` to select the gubernator peers from the endpoints listing") - } - } - - if anyHasPrefix("GUBER_MEMBERLIST_", os.Environ()) { - logrus.Debug("Memberlist pool config found") - if len(conf.MemberListPoolConf.KnownNodes) == 0 { - return conf, errors.New("when using `member-list` for peer discovery, you MUST provide a " + - "hostname of a known host in the cluster via `GUBER_MEMBERLIST_KNOWN_NODES`") - } - } - - if anyHasPrefix("GUBER_ETCD_", os.Environ()) { - logrus.Debug("ETCD peer pool config found") - } - - // If env contains any TLS configuration - if anyHasPrefix("GUBER_ETCD_TLS_", os.Environ()) { - if err := setupTLS(conf.EtcdPoolConf.EtcdConfig); err != nil { - return conf, err - } - } - - if gubernator.DebugEnabled { - spew.Dump(conf) - } - - return conf, nil -} - func checkErr(err error, msg string) { if err != nil { log.WithError(err).Error(msg) os.Exit(1) } } - -func setupTLS(conf *etcd.Config) error { - var tlsCertFile, tlsKeyFile, tlsCAFile string - - // set `GUBER_ETCD_TLS_ENABLE` and this line will - // create a TLS config with no config. - setter.SetDefault(&conf.TLS, &tls.Config{}) - - setter.SetDefault(&tlsCertFile, os.Getenv("GUBER_ETCD_TLS_CERT")) - setter.SetDefault(&tlsKeyFile, os.Getenv("GUBER_ETCD_TLS_KEY")) - setter.SetDefault(&tlsCAFile, os.Getenv("GUBER_ETCD_TLS_CA")) - - // If the CA file was provided - if tlsCAFile != "" { - setter.SetDefault(&conf.TLS, &tls.Config{}) - - var certPool *x509.CertPool = nil - if pemBytes, err := ioutil.ReadFile(tlsCAFile); err == nil { - certPool = x509.NewCertPool() - certPool.AppendCertsFromPEM(pemBytes) - } else { - return errors.Wrapf(err, "while loading cert CA file '%s'", tlsCAFile) - } - setter.SetDefault(&conf.TLS.RootCAs, certPool) - conf.TLS.InsecureSkipVerify = false - } - - // If the cert and key files are provided attempt to load them - if tlsCertFile != "" && tlsKeyFile != "" { - tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) - if err != nil { - return errors.Wrapf(err, "while loading cert '%s' and key file '%s'", - tlsCertFile, tlsKeyFile) - } - setter.SetDefault(&conf.TLS.Certificates, []tls.Certificate{tlsCert}) - } - - // If no other TLS config is provided this will force connecting with TLS, - // without cert verification - if os.Getenv("GUBER_ETCD_TLS_SKIP_VERIFY") != "" { - setter.SetDefault(&conf.TLS, &tls.Config{}) - conf.TLS.InsecureSkipVerify = true - } - return nil -} - -func anyHasPrefix(prefix string, items []string) bool { - for _, i := range items { - if strings.HasPrefix(i, prefix) { - return true - } - } - return false -} - -func getEnvBool(name string) bool { - v := os.Getenv(name) - if v == "" { - return false - } - b, err := strconv.ParseBool(v) - if err != nil { - log.WithError(err).Errorf("while parsing '%s' as an boolean", name) - return false - } - return b -} - -func getEnvInteger(name string) int { - v := os.Getenv(name) - if v == "" { - return 0 - } - i, err := strconv.ParseInt(v, 10, 64) - if err != nil { - log.WithError(err).Errorf("while parsing '%s' as an integer", name) - return 0 - } - return int(i) -} - -func getEnvDuration(name string) clock.Duration { - v := os.Getenv(name) - if v == "" { - return 0 - } - d, err := clock.ParseDuration(v) - if err != nil { - log.WithError(err).Errorf("while parsing '%s' as a duration", name) - return 0 - } - return d -} - -func getEnvSlice(name string) []string { - v := os.Getenv(name) - if v == "" { - return nil - } - return strings.Split(v, ",") -} - -// Take values from a file in the format `GUBER_CONF_ITEM=my-value` and put them into the environment -// lines that begin with `#` are ignored -func fromEnvFile(configFile string) error { - fd, err := os.Open(configFile) - if err != nil { - return fmt.Errorf("while opening config file: %s", err) - } - - contents, err := ioutil.ReadAll(fd) - if err != nil { - return fmt.Errorf("while reading config file '%s': %s", configFile, err) - } - for i, line := range strings.Split(string(contents), "\n") { - // Skip comments, empty lines or lines with tabs - if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " ") || - strings.HasPrefix(line, "\t") || len(line) == 0 { - continue - } - - logrus.Debugf("config: [%d] '%s'", i, line) - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - return errors.Errorf("malformed key=value on line '%d'", i) - } - - if err := os.Setenv(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])); err != nil { - return errors.Wrapf(err, "while settings environ for '%s=%s'", parts[0], parts[1]) - } - } - return nil -} - -func validClientAuthTypes(m map[string]tls.ClientAuthType) string { - var rs []string - for k, _ := range m { - rs = append(rs, k) - } - return strings.Join(rs, ",") -} - -func validHashKeys(m map[string]gubernator.HashFunc) string { - var rs []string - for k, _ := range m { - rs = append(rs, k) - } - return strings.Join(rs, ",") -} - -func validHash64Keys(m map[string]gubernator.HashFunc64) string { - var rs []string - for k, _ := range m { - rs = append(rs, k) - } - return strings.Join(rs, ",") -} diff --git a/config.go b/config.go index a4a4e126..b74fe01c 100644 --- a/config.go +++ b/config.go @@ -18,14 +18,50 @@ package gubernator import ( "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" + "net" + "os" + "strconv" + "strings" "time" + etcd "github.com/coreos/etcd/clientv3" + "github.com/davecgh/go-spew/spew" + "github.com/mailgun/holster/v3/clock" "github.com/mailgun/holster/v3/setter" + "github.com/mailgun/holster/v3/slice" + "github.com/pkg/errors" + "github.com/segmentio/fasthash/fnv1" + "github.com/segmentio/fasthash/fnv1a" "github.com/sirupsen/logrus" "google.golang.org/grpc" ) +type BehaviorConfig struct { + // How long we should wait for a batched response from a peer + BatchTimeout time.Duration + // How long we should wait before sending a batched request + BatchWait time.Duration + // The max number of requests we can batch into a single peer request + BatchLimit int + + // How long a non-owning peer should wait before syncing hits to the owning peer + GlobalSyncWait time.Duration + // How long we should wait for global sync responses from peers + GlobalTimeout time.Duration + // The max number of global updates we can batch into a single peer request + GlobalBatchLimit int + + // How long the current region will collect request before pushing them to other regions + MultiRegionSyncWait time.Duration + // How long the current region will wait for responses from other regions + MultiRegionTimeout time.Duration + // The max number of requests the current region will collect + MultiRegionBatchLimit int +} + // config for a gubernator instance type Config struct { // Required @@ -67,27 +103,33 @@ type Config struct { PeerTLS *tls.Config } -type BehaviorConfig struct { - // How long we should wait for a batched response from a peer - BatchTimeout time.Duration - // How long we should wait before sending a batched request - BatchWait time.Duration - // The max number of requests we can batch into a single peer request - BatchLimit int +func (c *Config) SetDefaults() error { + setter.SetDefault(&c.Behaviors.BatchTimeout, time.Millisecond*500) + setter.SetDefault(&c.Behaviors.BatchLimit, maxBatchSize) + setter.SetDefault(&c.Behaviors.BatchWait, time.Microsecond*500) - // How long a non-owning peer should wait before syncing hits to the owning peer - GlobalSyncWait time.Duration - // How long we should wait for global sync responses from peers - GlobalTimeout time.Duration - // The max number of global updates we can batch into a single peer request - GlobalBatchLimit int + setter.SetDefault(&c.Behaviors.GlobalTimeout, time.Millisecond*500) + setter.SetDefault(&c.Behaviors.GlobalBatchLimit, maxBatchSize) + setter.SetDefault(&c.Behaviors.GlobalSyncWait, time.Microsecond*500) - // How long the current region will collect request before pushing them to other regions - MultiRegionSyncWait time.Duration - // How long the current region will wait for responses from other regions - MultiRegionTimeout time.Duration - // The max number of requests the current region will collect - MultiRegionBatchLimit int + setter.SetDefault(&c.Behaviors.MultiRegionTimeout, time.Millisecond*500) + setter.SetDefault(&c.Behaviors.MultiRegionBatchLimit, maxBatchSize) + setter.SetDefault(&c.Behaviors.MultiRegionSyncWait, time.Second) + + setter.SetDefault(&c.LocalPicker, NewReplicatedConsistentHash(nil, DefaultReplicas)) + setter.SetDefault(&c.RegionPicker, NewRegionPicker(nil)) + setter.SetDefault(&c.Cache, NewLRUCache(0)) + + if c.Behaviors.BatchLimit > maxBatchSize { + return fmt.Errorf("Behaviors.BatchLimit cannot exceed '%d'", maxBatchSize) + } + + // Make a copy of the TLS config in case our caller decides to make changes + if c.PeerTLS != nil { + c.PeerTLS = c.PeerTLS.Clone() + } + + return nil } type PeerInfo struct { @@ -108,31 +150,397 @@ func (p PeerInfo) HashKey() string { type UpdateFunc func([]PeerInfo) -func (c *Config) SetDefaults() error { - setter.SetDefault(&c.Behaviors.BatchTimeout, time.Millisecond*500) - setter.SetDefault(&c.Behaviors.BatchLimit, maxBatchSize) - setter.SetDefault(&c.Behaviors.BatchWait, time.Microsecond*500) +var DebugEnabled = false - setter.SetDefault(&c.Behaviors.GlobalTimeout, time.Millisecond*500) - setter.SetDefault(&c.Behaviors.GlobalBatchLimit, maxBatchSize) - setter.SetDefault(&c.Behaviors.GlobalSyncWait, time.Microsecond*500) +type DaemonConfig struct { + // (Required) The `address:port` that will accept GRPC requests + GRPCListenAddress string - setter.SetDefault(&c.Behaviors.MultiRegionTimeout, time.Millisecond*500) - setter.SetDefault(&c.Behaviors.MultiRegionBatchLimit, maxBatchSize) - setter.SetDefault(&c.Behaviors.MultiRegionSyncWait, time.Second) + // (Required) The `address:port` that will accept HTTP requests + HTTPListenAddress string - setter.SetDefault(&c.LocalPicker, NewReplicatedConsistentHash(nil, DefaultReplicas)) - setter.SetDefault(&c.RegionPicker, NewRegionPicker(nil)) - setter.SetDefault(&c.Cache, NewLRUCache(0)) + // (Optional) The `address:port` that is advertised to other Gubernator peers. + // Defaults to `GRPCListenAddress` + AdvertiseAddress string - if c.Behaviors.BatchLimit > maxBatchSize { - return fmt.Errorf("Behaviors.BatchLimit cannot exceed '%d'", maxBatchSize) + // (Optional) The number of items in the cache. Defaults to 50,000 + CacheSize int + + // (Optional) Configure how behaviours behave + Behaviors BehaviorConfig + + // (Optional) Identifies the datacenter this instance is running in. For + // use with multi-region support + DataCenter string + + // (Optional) Which pool to use when discovering other Gubernator peers + // Valid options are [etcd, k8s, member-list] (Defaults to 'member-list') + PeerDiscoveryType string + + // (Optional) Etcd configuration used for peer discovery + EtcdPoolConf EtcdPoolConfig + + // (Optional) K8s configuration used for peer discovery + K8PoolConf K8sPoolConfig + + // (Optional) Member list configuration used for peer discovery + MemberListPoolConf MemberListPoolConfig + + // (Optional) The PeerPicker as selected by `GUBER_PEER_PICKER` + Picker PeerPicker + + // (Optional) A Logger which implements the declared logger interface (typically *logrus.Entry) + Logger logrus.FieldLogger + + // (Optional) TLS Configuration; SpawnDaemon() will modify the passed TLS config in an + // attempt to build a complete TLS config if one is not provided. + TLS *TLSConfig +} + +func (d *DaemonConfig) ClientTLS() *tls.Config { + if d.TLS != nil { + return d.TLS.ClientTLS } + return nil +} - // Make a copy of the TLS config in case our caller decides to make changes - if c.PeerTLS != nil { - c.PeerTLS = c.PeerTLS.Clone() +func (d *DaemonConfig) ServerTLS() *tls.Config { + if d.TLS != nil { + return d.TLS.ServerTLS + } + return nil +} + +// SetupDaemonConfig returns a DaemonConfig object as configured by reading the provided config file +// and environment. +func SetupDaemonConfig(logger *logrus.Logger, configFile string) (DaemonConfig, error) { + log := logrus.NewEntry(logger) + var conf DaemonConfig + + if configFile != "" { + log.Infof("Loading env config: %s", configFile) + if err := fromEnvFile(log, configFile); err != nil { + return conf, err + } + } + + setter.SetDefault(&DebugEnabled, getEnvBool(log, "GUBER_DEBUG")) + if DebugEnabled { + logger.SetLevel(logrus.DebugLevel) + log.Debug("Debug enabled") + } + + // Main config + setter.SetDefault(&conf.GRPCListenAddress, os.Getenv("GUBER_GRPC_ADDRESS"), "localhost:81") + setter.SetDefault(&conf.HTTPListenAddress, os.Getenv("GUBER_HTTP_ADDRESS"), "localhost:80") + setter.SetDefault(&conf.CacheSize, getEnvInteger(log, "GUBER_CACHE_SIZE"), 50_000) + setter.SetDefault(&conf.AdvertiseAddress, os.Getenv("GUBER_ADVERTISE_ADDRESS"), conf.GRPCListenAddress) + setter.SetDefault(&conf.DataCenter, os.Getenv("GUBER_DATA_CENTER"), "") + + advAddr, advPort, err := net.SplitHostPort(conf.AdvertiseAddress) + if err != nil { + return conf, errors.Wrap(err, "GUBER_ADVERTISE_ADDRESS is invalid; expected format is `address:port`") + } + advAddr, err = ResolveHostIP(advAddr) + if err != nil { + return conf, errors.Wrap(err, "failed to discover host ip for GUBER_ADVERTISE_ADDRESS") + } + conf.AdvertiseAddress = net.JoinHostPort(advAddr, advPort) + + // Behaviors + setter.SetDefault(&conf.Behaviors.BatchTimeout, getEnvDuration(log, "GUBER_BATCH_TIMEOUT")) + setter.SetDefault(&conf.Behaviors.BatchLimit, getEnvInteger(log, "GUBER_BATCH_LIMIT")) + setter.SetDefault(&conf.Behaviors.BatchWait, getEnvDuration(log, "GUBER_BATCH_WAIT")) + + setter.SetDefault(&conf.Behaviors.GlobalTimeout, getEnvDuration(log, "GUBER_GLOBAL_TIMEOUT")) + setter.SetDefault(&conf.Behaviors.GlobalBatchLimit, getEnvInteger(log, "GUBER_GLOBAL_BATCH_LIMIT")) + setter.SetDefault(&conf.Behaviors.GlobalSyncWait, getEnvDuration(log, "GUBER_GLOBAL_SYNC_WAIT")) + + setter.SetDefault(&conf.Behaviors.MultiRegionTimeout, getEnvDuration(log, "GUBER_MULTI_REGION_TIMEOUT")) + setter.SetDefault(&conf.Behaviors.MultiRegionBatchLimit, getEnvInteger(log, "GUBER_MULTI_REGION_BATCH_LIMIT")) + setter.SetDefault(&conf.Behaviors.MultiRegionSyncWait, getEnvDuration(log, "GUBER_MULTI_REGION_SYNC_WAIT")) + + choices := []string{"member-list", "k8s", "etcd"} + setter.SetDefault(&conf.PeerDiscoveryType, os.Getenv("GUBER_PEER_DISCOVERY_TYPE"), "member-list") + if !slice.ContainsString(conf.PeerDiscoveryType, choices, nil) { + return conf, fmt.Errorf("GUBER_PEER_DISCOVERY_TYPE is invalid; choices are [%s]`", strings.Join(choices, ",")) + } + + // TLS Config + setter.SetDefault(&conf.TLS, &TLSConfig{}) + setter.SetDefault(&conf.TLS.CaFile, os.Getenv("GUBER_TLS_CA")) + setter.SetDefault(&conf.TLS.CaKeyFile, os.Getenv("GUBER_TLS_CA_KEY")) + setter.SetDefault(&conf.TLS.KeyFile, os.Getenv("GUBER_TLS_KEY")) + setter.SetDefault(&conf.TLS.CertFile, os.Getenv("GUBER_TLS_CERT")) + setter.SetDefault(&conf.TLS.AutoTLS, getEnvBool(log, "GUBER_TLS_AUTO")) + + clientAuth := os.Getenv("GUBER_TLS_CLIENT_AUTH") + if clientAuth != "" { + clientAuthTypes := map[string]tls.ClientAuthType{ + "request-cert": tls.RequestClientCert, + "verify-cert": tls.VerifyClientCertIfGiven, + "require-any-cert": tls.RequireAnyClientCert, + "require-and-verify": tls.RequireAndVerifyClientCert, + } + t, ok := clientAuthTypes[clientAuth] + if !ok { + return conf, errors.Errorf("'GUBER_TLS_CLIENT_AUTH=%s' is invalid; choices are [%s]", + clientAuth, validClientAuthTypes(clientAuthTypes)) + } + conf.TLS.ClientAuth = t + } + setter.SetDefault(&conf.TLS.ClientAuthKeyFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_KEY")) + setter.SetDefault(&conf.TLS.ClientAuthCertFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_CERT")) + setter.SetDefault(&conf.TLS.ClientAuthCaFile, os.Getenv("GUBER_TLS_CLIENT_AUTH_CA_CERT")) + setter.SetDefault(&conf.TLS.InsecureSkipVerify, getEnvBool(log, "GUBER_TLS_INSECURE_SKIP_VERIFY")) + + // ETCD Config + setter.SetDefault(&conf.EtcdPoolConf.KeyPrefix, os.Getenv("GUBER_ETCD_KEY_PREFIX"), "/gubernator-peers") + setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig, &etcd.Config{}) + setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Endpoints, getEnvSlice("GUBER_ETCD_ENDPOINTS"), []string{"localhost:2379"}) + setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.DialTimeout, getEnvDuration(log, "GUBER_ETCD_DIAL_TIMEOUT"), clock.Second*5) + setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Username, os.Getenv("GUBER_ETCD_USER")) + setter.SetDefault(&conf.EtcdPoolConf.EtcdConfig.Password, os.Getenv("GUBER_ETCD_PASSWORD")) + setter.SetDefault(&conf.EtcdPoolConf.AdvertiseAddress, os.Getenv("GUBER_ETCD_ADVERTISE_ADDRESS"), conf.AdvertiseAddress) + setter.SetDefault(&conf.EtcdPoolConf.DataCenter, os.Getenv("GUBER_ETCD_DATA_CENTER"), conf.DataCenter) + + setter.SetDefault(&conf.MemberListPoolConf.AdvertiseAddress, os.Getenv("GUBER_MEMBERLIST_ADVERTISE_ADDRESS"), conf.AdvertiseAddress) + setter.SetDefault(&conf.MemberListPoolConf.MemberListAddress, os.Getenv("GUBER_MEMBERLIST_ADDRESS"), fmt.Sprintf("%s:7946", advAddr)) + setter.SetDefault(&conf.MemberListPoolConf.KnownNodes, getEnvSlice("GUBER_MEMBERLIST_KNOWN_NODES"), []string{}) + setter.SetDefault(&conf.MemberListPoolConf.DataCenter, conf.DataCenter) + + // Kubernetes Config + setter.SetDefault(&conf.K8PoolConf.Namespace, os.Getenv("GUBER_K8S_NAMESPACE"), "default") + conf.K8PoolConf.PodIP = os.Getenv("GUBER_K8S_POD_IP") + conf.K8PoolConf.PodPort = os.Getenv("GUBER_K8S_POD_PORT") + conf.K8PoolConf.Selector = os.Getenv("GUBER_K8S_ENDPOINTS_SELECTOR") + + // PeerPicker Config + if pp := os.Getenv("GUBER_PEER_PICKER"); pp != "" { + var replicas int + var hash string + + switch pp { + case "consistent-hash": + setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a") + hashFuncs := map[string]HashFunc{ + "fnv1a": fnv1a.HashBytes32, + "fnv1": fnv1.HashBytes32, + "crc32": nil, + } + fn, ok := hashFuncs[hash] + if !ok { + return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]", + hash, validHashKeys(hashFuncs)) + } + conf.Picker = NewConsistentHash(fn) + + case "replicated-hash": + setter.SetDefault(&replicas, getEnvInteger(log, "GUBER_REPLICATED_HASH_REPLICAS"), DefaultReplicas) + conf.Picker = NewReplicatedConsistentHash(nil, replicas) + setter.SetDefault(&hash, os.Getenv("GUBER_PEER_PICKER_HASH"), "fnv1a") + hashFuncs := map[string]HashFunc64{ + "fnv1a": fnv1a.HashBytes64, + "fnv1": fnv1.HashBytes64, + } + fn, ok := hashFuncs[hash] + if !ok { + return conf, errors.Errorf("'GUBER_PEER_PICKER_HASH=%s' is invalid; choices are [%s]", + hash, validHash64Keys(hashFuncs)) + } + conf.Picker = NewReplicatedConsistentHash(fn, replicas) + default: + return conf, errors.Errorf("'GUBER_PEER_PICKER=%s' is invalid; choices are ['replicated-hash', 'consistent-hash']", pp) + } + } + + if anyHasPrefix("GUBER_K8S_", os.Environ()) { + log.Debug("K8s peer pool config found") + if conf.K8PoolConf.Selector == "" { + return conf, errors.New("when using k8s for peer discovery, you MUST provide a " + + "`GUBER_K8S_ENDPOINTS_SELECTOR` to select the gubernator peers from the endpoints listing") + } + } + + if anyHasPrefix("GUBER_MEMBERLIST_", os.Environ()) { + log.Debug("Memberlist pool config found") + if len(conf.MemberListPoolConf.KnownNodes) == 0 { + return conf, errors.New("when using `member-list` for peer discovery, you MUST provide a " + + "hostname of a known host in the cluster via `GUBER_MEMBERLIST_KNOWN_NODES`") + } + } + + if anyHasPrefix("GUBER_ETCD_", os.Environ()) { + log.Debug("ETCD peer pool config found") + } + + // If env contains any TLS configuration + if anyHasPrefix("GUBER_ETCD_TLS_", os.Environ()) { + if err := setupEtcdTLS(conf.EtcdPoolConf.EtcdConfig); err != nil { + return conf, err + } + } + + if DebugEnabled { + log.Debug(spew.Sdump(conf)) + } + + return conf, nil +} + +func setupEtcdTLS(conf *etcd.Config) error { + var tlsCertFile, tlsKeyFile, tlsCAFile string + + // set `GUBER_ETCD_TLS_ENABLE` and this line will + // create a TLS config with no config. + setter.SetDefault(&conf.TLS, &tls.Config{}) + + setter.SetDefault(&tlsCertFile, os.Getenv("GUBER_ETCD_TLS_CERT")) + setter.SetDefault(&tlsKeyFile, os.Getenv("GUBER_ETCD_TLS_KEY")) + setter.SetDefault(&tlsCAFile, os.Getenv("GUBER_ETCD_TLS_CA")) + + // If the CA file was provided + if tlsCAFile != "" { + setter.SetDefault(&conf.TLS, &tls.Config{}) + + var certPool *x509.CertPool = nil + if pemBytes, err := ioutil.ReadFile(tlsCAFile); err == nil { + certPool = x509.NewCertPool() + certPool.AppendCertsFromPEM(pemBytes) + } else { + return errors.Wrapf(err, "while loading cert CA file '%s'", tlsCAFile) + } + setter.SetDefault(&conf.TLS.RootCAs, certPool) + conf.TLS.InsecureSkipVerify = false + } + + // If the cert and key files are provided attempt to load them + if tlsCertFile != "" && tlsKeyFile != "" { + tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) + if err != nil { + return errors.Wrapf(err, "while loading cert '%s' and key file '%s'", + tlsCertFile, tlsKeyFile) + } + setter.SetDefault(&conf.TLS.Certificates, []tls.Certificate{tlsCert}) + } + + // If no other TLS config is provided this will force connecting with TLS, + // without cert verification + if os.Getenv("GUBER_ETCD_TLS_SKIP_VERIFY") != "" { + setter.SetDefault(&conf.TLS, &tls.Config{}) + conf.TLS.InsecureSkipVerify = true + } + return nil +} + +func anyHasPrefix(prefix string, items []string) bool { + for _, i := range items { + if strings.HasPrefix(i, prefix) { + return true + } + } + return false +} + +func getEnvBool(log logrus.FieldLogger, name string) bool { + v := os.Getenv(name) + if v == "" { + return false + } + b, err := strconv.ParseBool(v) + if err != nil { + log.WithError(err).Errorf("while parsing '%s' as an boolean", name) + return false + } + return b +} + +func getEnvInteger(log logrus.FieldLogger, name string) int { + v := os.Getenv(name) + if v == "" { + return 0 + } + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + log.WithError(err).Errorf("while parsing '%s' as an integer", name) + return 0 } + return int(i) +} + +func getEnvDuration(log logrus.FieldLogger, name string) time.Duration { + v := os.Getenv(name) + if v == "" { + return 0 + } + d, err := time.ParseDuration(v) + if err != nil { + log.WithError(err).Errorf("while parsing '%s' as a duration", name) + return 0 + } + return d +} + +func getEnvSlice(name string) []string { + v := os.Getenv(name) + if v == "" { + return nil + } + return strings.Split(v, ",") +} + +// Take values from a file in the format `GUBER_CONF_ITEM=my-value` and put them into the environment +// lines that begin with `#` are ignored +func fromEnvFile(log logrus.FieldLogger, configFile string) error { + fd, err := os.Open(configFile) + if err != nil { + return fmt.Errorf("while opening config file: %s", err) + } + + contents, err := ioutil.ReadAll(fd) + if err != nil { + return fmt.Errorf("while reading config file '%s': %s", configFile, err) + } + for i, line := range strings.Split(string(contents), "\n") { + // Skip comments, empty lines or lines with tabs + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, " ") || + strings.HasPrefix(line, "\t") || len(line) == 0 { + continue + } + log.Debugf("config: [%d] '%s'", i, line) + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return errors.Errorf("malformed key=value on line '%d'", i) + } + + if err := os.Setenv(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])); err != nil { + return errors.Wrapf(err, "while settings environ for '%s=%s'", parts[0], parts[1]) + } + } return nil } + +func validClientAuthTypes(m map[string]tls.ClientAuthType) string { + var rs []string + for k, _ := range m { + rs = append(rs, k) + } + return strings.Join(rs, ",") +} + +func validHashKeys(m map[string]HashFunc) string { + var rs []string + for k, _ := range m { + rs = append(rs, k) + } + return strings.Join(rs, ",") +} + +func validHash64Keys(m map[string]HashFunc64) string { + var rs []string + for k, _ := range m { + rs = append(rs, k) + } + return strings.Join(rs, ",") +} diff --git a/daemon.go b/daemon.go index 39e7cc9e..9b56714a 100644 --- a/daemon.go +++ b/daemon.go @@ -18,7 +18,6 @@ package gubernator import ( "context" - "crypto/tls" "log" "net" "net/http" @@ -36,67 +35,6 @@ import ( "google.golang.org/grpc/credentials" ) -var DebugEnabled = false - -type DaemonConfig struct { - // (Required) The `address:port` that will accept GRPC requests - GRPCListenAddress string - - // (Required) The `address:port` that will accept HTTP requests - HTTPListenAddress string - - // (Optional) The `address:port` that is advertised to other Gubernator peers. - // Defaults to `GRPCListenAddress` - AdvertiseAddress string - - // (Optional) The number of items in the cache. Defaults to 50,000 - CacheSize int - - // (Optional) Configure how behaviours behave - Behaviors BehaviorConfig - - // (Optional) Identifies the datacenter this instance is running in. For - // use with multi-region support - DataCenter string - - // (Optional) Which pool to use when discovering other Gubernator peers - // Valid options are [etcd, k8s, member-list] (Defaults to 'member-list') - PeerDiscoveryType string - - // (Optional) Etcd configuration used for peer discovery - EtcdPoolConf EtcdPoolConfig - - // (Optional) K8s configuration used for peer discovery - K8PoolConf K8sPoolConfig - - // (Optional) Member list configuration used for peer discovery - MemberListPoolConf MemberListPoolConfig - - // (Optional) The PeerPicker as selected by `GUBER_PEER_PICKER` - Picker PeerPicker - - // (Optional) A Logger which implements the declared logger interface (typically *logrus.Entry) - Logger logrus.FieldLogger - - // (Optional) TLS Configuration; SpawnDaemon() will modify the passed TLS config in an - // attempt to build a complete TLS config if one is not provided. - TLS *TLSConfig -} - -func (d *DaemonConfig) ClientTLS() *tls.Config { - if d.TLS != nil { - return d.TLS.ClientTLS - } - return nil -} - -func (d *DaemonConfig) ServerTLS() *tls.Config { - if d.TLS != nil { - return d.TLS.ServerTLS - } - return nil -} - type Daemon struct { GRPCListener net.Listener HTTPListener net.Listener diff --git a/docker-compose-tls.yaml b/docker-compose-tls.yaml index 2a8e57a6..3cd3df47 100644 --- a/docker-compose-tls.yaml +++ b/docker-compose-tls.yaml @@ -10,15 +10,15 @@ services: - GUBER_ADVERTISE_ADDRESS=gubernator-1:81 - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 # TLS config - - GUBER_TLS_CA_FILE=/etc/tls-certs/ca.pem - - GUBER_TLS_KEY_FILE=/etc/tls-certs/gubernator.key - - GUBER_TLS_CERT_FILE=/etc/tls-certs/gubernator.pem + - GUBER_TLS_CA=/etc/tls/ca.pem + - GUBER_TLS_KEY=/etc/tls/gubernator.key + - GUBER_TLS_CERT=/etc/tls/gubernator.pem - GUBER_TLS_CLIENT_AUTH=require-and-verify ports: - "9081:81" - "9080:80" volumes: - - ${PWD}/certs:/etc/tls-certs + - ${PWD}/certs:/etc/tls gubernator-2: image: thrawn01/gubernator:latest @@ -30,15 +30,15 @@ services: - GUBER_ADVERTISE_ADDRESS=gubernator-2:81 - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 # TLS config - - GUBER_TLS_CA_FILE=/etc/tls-certs/ca.pem - - GUBER_TLS_KEY_FILE=/etc/tls-certs/gubernator.key - - GUBER_TLS_CERT_FILE=/etc/tls-certs/gubernator.pem + - GUBER_TLS_CA=/etc/tls/ca.pem + - GUBER_TLS_KEY=/etc/tls/gubernator.key + - GUBER_TLS_CERT=/etc/tls/gubernator.pem - GUBER_TLS_CLIENT_AUTH=require-and-verify ports: - "9181:81" - "9180:80" volumes: - - ${PWD}/certs:/etc/tls-certs + - ${PWD}/certs:/etc/tls gubernator-3: image: thrawn01/gubernator:latest @@ -50,15 +50,15 @@ services: - GUBER_ADVERTISE_ADDRESS=gubernator-3:81 - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 # TLS config - - GUBER_TLS_CA_FILE=/etc/tls-certs/ca.pem - - GUBER_TLS_KEY_FILE=/etc/tls-certs/gubernator.key - - GUBER_TLS_CERT_FILE=/etc/tls-certs/gubernator.pem + - GUBER_TLS_CA=/etc/tls/ca.pem + - GUBER_TLS_KEY=/etc/tls/gubernator.key + - GUBER_TLS_CERT=/etc/tls/gubernator.pem - GUBER_TLS_CLIENT_AUTH=require-and-verify ports: - "9281:81" - "9280:80" volumes: - - ${PWD}/certs:/etc/tls-certs + - ${PWD}/certs:/etc/tls gubernator-4: image: thrawn01/gubernator:latest @@ -71,12 +71,12 @@ services: - GUBER_ADVERTISE_ADDRESS=gubernator-4:81 - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 # TLS config - - GUBER_TLS_CA_FILE=/etc/tls-certs/ca.pem - - GUBER_TLS_KEY_FILE=/etc/tls-certs/gubernator.key - - GUBER_TLS_CERT_FILE=/etc/tls-certs/gubernator.pem + - GUBER_TLS_CA=/etc/tls/ca.pem + - GUBER_TLS_KEY=/etc/tls/gubernator.key + - GUBER_TLS_CERT=/etc/tls/gubernator.pem - GUBER_TLS_CLIENT_AUTH=require-and-verify ports: - "9381:81" - "9380:80" volumes: - - ${PWD}/certs:/etc/tls-certs + - ${PWD}/certs:/etc/tls diff --git a/docker-compose.yaml b/docker-compose.yaml index cac65026..9dba583e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,8 +10,6 @@ services: - GUBER_HTTP_ADDRESS=0.0.0.0:80 # The address that is advertised to other peers - GUBER_ADVERTISE_ADDRESS=gubernator-1:81 - # Max size of the cache; The cache size will never grow beyond this size. - - GUBER_CACHE_SIZE=50000 # A comma separated list of known gubernator nodes - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 #- GUBER_DATA_CENTER=us-east-1 @@ -29,8 +27,6 @@ services: - GUBER_HTTP_ADDRESS=0.0.0.0:80 # The address that is advertised to other peers - GUBER_ADVERTISE_ADDRESS=gubernator-2:81 - # Max size of the cache; The cache size will never grow beyond this size. - - GUBER_CACHE_SIZE=50000 # A comma separated list of known gubernator nodes - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 #- GUBER_DATA_CENTER=us-east-1 @@ -48,8 +44,6 @@ services: - GUBER_HTTP_ADDRESS=0.0.0.0:80 # The address that is advertised to other peers - GUBER_ADVERTISE_ADDRESS=gubernator-3:81 - # Max size of the cache; The cache size will never grow beyond this size. - - GUBER_CACHE_SIZE=50000 # A comma separated list of known gubernator nodes - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 #- GUBER_DATA_CENTER=us-west-2 @@ -68,10 +62,8 @@ services: - GUBER_HTTP_ADDRESS=0.0.0.0:80 # The address that is advertised to other peers - GUBER_ADVERTISE_ADDRESS=gubernator-4:81 - # Max size of the cache; The cache size will never grow beyond this size. - - GUBER_CACHE_SIZE=50000 # A Comma separated list of known gubernator nodes - - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1 + - GUBER_MEMBERLIST_KNOWN_NODES=gubernator-1,gubernator-2 #- GUBER_DATA_CENTER=us-west-2 ports: - "9381:81" diff --git a/k8s-deployment.yaml b/k8s-deployment.yaml index 8a45cf0b..3b661bb5 100644 --- a/k8s-deployment.yaml +++ b/k8s-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: gubernator spec: - replicas: 2 + replicas: 4 selector: matchLabels: app: gubernator @@ -33,6 +33,9 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + # Use the k8s API for peer discovery + - name: GUBER_PEER_DISCOVERY_TYPE + value: "k8s" # This should match the port number GRPC is listening on # as defined by `containerPort` - name: GUBER_K8S_POD_PORT @@ -42,8 +45,8 @@ spec: - name: GUBER_K8S_ENDPOINTS_SELECTOR value: "app=gubernator" # Enable debug for diagnosing issues - #- name: GUBER_DEBUG - # value: "true" + - name: GUBER_DEBUG + value: "true" restartPolicy: Always --- apiVersion: v1 diff --git a/kubernetes.go b/kubernetes.go index aafd8206..153628c9 100644 --- a/kubernetes.go +++ b/kubernetes.go @@ -94,7 +94,7 @@ func (e *K8sPool) start() error { e.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) - e.log.Debugf("Queue (Add) '%s' - %s", key, err) + e.log.Debugf("Queue (Add) '%s' - %v", key, err) if err != nil { e.log.Errorf("while calling MetaNamespaceKeyFunc(): %s", err) return @@ -102,7 +102,7 @@ func (e *K8sPool) start() error { }, UpdateFunc: func(obj, new interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) - e.log.Debugf("Queue (Update) '%s' - %s", key, err) + e.log.Debugf("Queue (Update) '%s' - %v", key, err) if err != nil { e.log.Errorf("while calling MetaNamespaceKeyFunc(): %s", err) return @@ -111,7 +111,7 @@ func (e *K8sPool) start() error { }, DeleteFunc: func(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) - e.log.Debugf("Queue (Delete) '%s' - %s", key, err) + e.log.Debugf("Queue (Delete) '%s' - %v", key, err) if err != nil { e.log.Errorf("while calling MetaNamespaceKeyFunc(): %s", err) return diff --git a/tls.go b/tls.go index 75032a6b..3b967860 100644 --- a/tls.go +++ b/tls.go @@ -118,7 +118,7 @@ func fromFile(name string) (*bytes.Buffer, error) { func SetupTLS(conf *TLSConfig) error { var err error - if conf == nil || conf.ServerTLS != nil { + if conf == nil || conf.ServerTLS == nil { return nil } diff --git a/version b/version index 9c218192..3c029ddf 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.0.0-rc.1 +1.0.0-rc.3