feat: add opentracing + jaeger

This adds support for opentracing with Jaeger. It is disabled by default, and can be turned on with a configurable sample rate. The bulk of the changes here are adding `context.Context` as the first parameter to the internal functions to allow for span context propogation.

This does change the protocol, adding a new optional field to allow for span propogation between keyless instances. #276 allows this.

Author: nickysemenza

client/client.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"crypto"
 	"crypto/ecdsa"
 	"crypto/rsa"
@@ -18,7 +19,9 @@ import (
 
 	"github.com/cloudflare/cfssl/log"
 	"github.com/cloudflare/gokeyless/protocol"
+	"github.com/cloudflare/gokeyless/tracing"
 	"github.com/lziest/ttlcache"
+	"github.com/opentracing/opentracing-go"
 )
 
 const (
@@ -190,8 +193,10 @@ func (c *Client) getRemote(server string) (Remote, error) {
 // NewRemoteSignerWithCertID returns a remote keyserver based crypto.Signer
 // ski, sni, serverIP, and certID are used to identify the key by the remote
 // keyserver.
-func NewRemoteSignerWithCertID(c *Client, keyserver string, ski protocol.SKI,
+func NewRemoteSignerWithCertID(ctx context.Context, c *Client, keyserver string, ski protocol.SKI,
 	pub crypto.PublicKey, sni string, certID string, serverIP net.IP) (crypto.Signer, error) {
+	span, _ := opentracing.StartSpanFromContext(ctx, "client.NewRemoteSignerWithCertID")
+	defer span.Finish()
 	priv := PrivateKey{
 		public:    pub,
 		client:    c,
@@ -201,6 +206,11 @@ func NewRemoteSignerWithCertID(c *Client, keyserver string, ski protocol.SKI,
 		keyserver: keyserver,
 		certID:    certID,
 	}
+	var err error
+	priv.JaegerSpan, err = tracing.SpanContextToBinary(span.Context())
+	if err != nil {
+		log.Errorf("failed to inject span: %v", err)
+	}
 
 	// This is due to an issue in crypto/tls, where an ECDSA key is not allowed to
 	// implement Decrypt.
@@ -213,8 +223,11 @@ func NewRemoteSignerWithCertID(c *Client, keyserver string, ski protocol.SKI,
 // NewRemoteSigner returns a remote keyserver based crypto.Signer,
 // ski, sni, and serverIP are used to identified the key by the remote
 // keyserver.
-func NewRemoteSigner(c *Client, keyserver string, ski protocol.SKI,
+func NewRemoteSigner(ctx context.Context, c *Client, keyserver string, ski protocol.SKI,
 	pub crypto.PublicKey, sni string, serverIP net.IP) (crypto.Signer, error) {
+
+	span, _ := opentracing.StartSpanFromContext(ctx, "client.NewRemoteSignerWithCertID")
+	defer span.Finish()
 	priv := PrivateKey{
 		public:    pub,
 		client:    c,
@@ -223,6 +236,11 @@ func NewRemoteSigner(c *Client, keyserver string, ski protocol.SKI,
 		serverIP:  serverIP,
 		keyserver: keyserver,
 	}
+	var err error
+	priv.JaegerSpan, err = tracing.SpanContextToBinary(span.Context())
+	if err != nil {
+		log.Errorf("failed to inject span: %v", err)
+	}
 
 	// This is due to an issue in crypto/tls, where an ECDSA key is not allowed to
 	// implement Decrypt.
@@ -237,42 +255,42 @@ func NewRemoteSigner(c *Client, keyserver string, ski protocol.SKI,
 // SKI is computed from the public key and along with sni and serverIP,
 // the remote Signer uses those key identification info to contact the
 // remote keyserver for keyless operations.
-func (c *Client) NewRemoteSignerTemplate(keyserver string, pub crypto.PublicKey, sni string, serverIP net.IP) (crypto.Signer, error) {
+func (c *Client) NewRemoteSignerTemplate(ctx context.Context, keyserver string, pub crypto.PublicKey, sni string, serverIP net.IP) (crypto.Signer, error) {
 	ski, err := protocol.GetSKI(pub)
 	if err != nil {
 		return nil, err
 	}
-	return NewRemoteSigner(c, keyserver, ski, pub, sni, serverIP)
+	return NewRemoteSigner(ctx, c, keyserver, ski, pub, sni, serverIP)
 }
 
 // NewRemoteSignerTemplateWithCertID returns a remote keyserver
 // based crypto.Signer with the public key.
 // SKI is computed from public key, and along with sni, serverIP, and
 // certID the remote signer uses these to contact the remote keyserver.
-func (c *Client) NewRemoteSignerTemplateWithCertID(keyserver string, pub crypto.PublicKey, sni string, serverIP net.IP, certID string) (crypto.Signer, error) {
+func (c *Client) NewRemoteSignerTemplateWithCertID(ctx context.Context, keyserver string, pub crypto.PublicKey, sni string, serverIP net.IP, certID string) (crypto.Signer, error) {
 	ski, err := protocol.GetSKI(pub)
 	if err != nil {
 		return nil, err
 	}
-	return NewRemoteSignerWithCertID(c, keyserver, ski, pub, sni, certID, serverIP)
+	return NewRemoteSignerWithCertID(ctx, c, keyserver, ski, pub, sni, certID, serverIP)
 }
 
 // NewRemoteSignerByPublicKey returns a remote keyserver based signer
 // with the the public key.
-func (c *Client) NewRemoteSignerByPublicKey(server string, pub crypto.PublicKey) (crypto.Signer, error) {
-	return c.NewRemoteSignerTemplate(server, pub, "", nil)
+func (c *Client) NewRemoteSignerByPublicKey(ctx context.Context, server string, pub crypto.PublicKey) (crypto.Signer, error) {
+	return c.NewRemoteSignerTemplate(ctx, server, pub, "", nil)
 }
 
 // NewRemoteSignerByCert returns a remote keyserver based signer
 // with the the public key contained in a x509.Certificate.
-func (c *Client) NewRemoteSignerByCert(server string, cert *x509.Certificate) (crypto.Signer, error) {
-	return c.NewRemoteSignerTemplate(server, cert.PublicKey, "", nil)
+func (c *Client) NewRemoteSignerByCert(ctx context.Context, server string, cert *x509.Certificate) (crypto.Signer, error) {
+	return c.NewRemoteSignerTemplate(ctx, server, cert.PublicKey, "", nil)
 }
 
 // NewRemoteSignerByCertPEM returns a remote keyserver based signer
 // with the public key extracted from  a single PEM cert
 // (possibly the leaf of a chain of certs).
-func (c *Client) NewRemoteSignerByCertPEM(server string, certsPEM []byte) (crypto.Signer, error) {
+func (c *Client) NewRemoteSignerByCertPEM(ctx context.Context, server string, certsPEM []byte) (crypto.Signer, error) {
 	block, _ := pem.Decode(certsPEM)
 	if block == nil {
 		return nil, errors.New("couldn't parse PEM bytes")
@@ -283,7 +301,7 @@ func (c *Client) NewRemoteSignerByCertPEM(server string, certsPEM []byte) (crypt
 		return nil, err
 	}
 
-	return c.NewRemoteSignerTemplate(server, cert.PublicKey, "", nil)
+	return c.NewRemoteSignerTemplate(ctx, server, cert.PublicKey, "", nil)
 }
 
 var (
@@ -318,11 +336,11 @@ func (c *Client) ScanDir(server, dir string, LoadPubKey func([]byte) (crypto.Pub
 					return err
 				}
 
-				if priv, err = c.NewRemoteSignerByPublicKey(server, pub); err != nil {
+				if priv, err = c.NewRemoteSignerByPublicKey(context.Background(), server, pub); err != nil {
 					return err
 				}
 			} else {
-				if priv, err = c.NewRemoteSignerByCertPEM(server, in); err != nil {
+				if priv, err = c.NewRemoteSignerByCertPEM(context.Background(), server, in); err != nil {
 					return err
 				}
 			}
@@ -363,7 +381,7 @@ func (c *Client) LoadTLSCertificate(server, certFile string) (cert tls.Certifica
 		return fail(err)
 	}
 
-	cert.PrivateKey, err = c.NewRemoteSignerByCert(server, cert.Leaf)
+	cert.PrivateKey, err = c.NewRemoteSignerByCert(context.TODO(), server, cert.Leaf)
 	if err != nil {
 		return fail(err)
 	}

client/keys.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"crypto"
 	"crypto/ecdsa"
 	"crypto/rsa"
@@ -12,6 +13,9 @@ import (
 
 	"github.com/cloudflare/cfssl/log"
 	"github.com/cloudflare/gokeyless/protocol"
+	"github.com/cloudflare/gokeyless/tracing"
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 	"golang.org/x/crypto/ed25519"
 )
 
@@ -87,6 +91,10 @@ type PrivateKey struct {
 	keyserver string
 	sni       string
 	certID    string
+
+	// We have shove the span context inside PrivateKey because
+	// it's used by calling functions on the `crypto.Signer` interface, which don't take ctx as a parameter.
+	JaegerSpan []byte
 }
 
 // Public returns the public key corresponding to the opaque private key.
@@ -96,7 +104,9 @@ func (key *PrivateKey) Public() crypto.PublicKey {
 
 // execute performs an opaque cryptographic operation on a server associated
 // with the key.
-func (key *PrivateKey) execute(op protocol.Op, msg []byte) ([]byte, error) {
+func (key *PrivateKey) execute(ctx context.Context, op protocol.Op, msg []byte) ([]byte, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PrivateKey.execute")
+	defer span.Finish()
 	var result *protocol.Operation
 	// retry once if connection returned by remote Dial is problematic.
 	for attempts := 2; attempts > 0; attempts-- {
@@ -110,7 +120,11 @@ func (key *PrivateKey) execute(op protocol.Op, msg []byte) ([]byte, error) {
 			return nil, err
 		}
 
-		result, err = conn.Conn.DoOperation(protocol.Operation{
+		// We explicitly do NOT want to fill in JaegerSpan here, since the remote keyless server
+		// will error if it does know how to handle that Tag
+		// https://github.com/cloudflare/gokeyless/pull/276 makes it safe to fill it in,
+		// but there's no way to know the version of the remote keyserver
+		result, err = conn.Conn.DoOperation(ctx, protocol.Operation{
 			Opcode:   op,
 			Payload:  msg,
 			SKI:      key.ski,
@@ -149,6 +163,13 @@ func (key *PrivateKey) execute(op protocol.Op, msg []byte) ([]byte, error) {
 
 // Sign implements the crypto.Signer operation for the given key.
 func (key *PrivateKey) Sign(r io.Reader, msg []byte, opts crypto.SignerOpts) ([]byte, error) {
+	spanCtx, err := tracing.SpanContextFromBinary(key.JaegerSpan)
+	if err != nil {
+		log.Errorf("failed to extract span: %v", err)
+	}
+	span, ctx := opentracing.StartSpanFromContext(context.Background(), "client: PrivateKey.Sign", ext.RPCServerOption(spanCtx))
+	defer span.Finish()
+
 	// If opts specifies a hash function, then the message is expected to be the
 	// length of the output of that hash function.
 	if opts.HashFunc() != 0 && len(msg) != opts.HashFunc().Size() {
@@ -159,7 +180,7 @@ func (key *PrivateKey) Sign(r io.Reader, msg []byte, opts crypto.SignerOpts) ([]
 	if op == protocol.OpError {
 		return nil, errors.New("invalid key type, hash or options")
 	}
-	return key.execute(op, msg)
+	return key.execute(ctx, op, msg)
 }
 
 // Decrypter implements the Decrypt method on a PrivateKey.
@@ -169,12 +190,18 @@ type Decrypter struct {
 
 // Decrypt implements the crypto.Decrypter operation for the given key.
 func (key *Decrypter) Decrypt(rand io.Reader, msg []byte, opts crypto.DecrypterOpts) ([]byte, error) {
+	spanCtx, err := tracing.SpanContextFromBinary(key.JaegerSpan)
+	if err != nil {
+		log.Errorf("failed to extract span: %v", err)
+	}
+	span, ctx := opentracing.StartSpanFromContext(context.Background(), "client: Decrypter.Decrypt", ext.RPCServerOption(spanCtx))
+	defer span.Finish()
 	opts1v15, ok := opts.(*rsa.PKCS1v15DecryptOptions)
 	if opts != nil && !ok {
 		return nil, errors.New("invalid options for Decrypt")
 	}
 
-	ptxt, err := key.execute(protocol.OpRSADecrypt, msg)
+	ptxt, err := key.execute(ctx, protocol.OpRSADecrypt, msg)
 	if err != nil {
 		return nil, err
 	}

client/remote.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"fmt"
@@ -120,7 +121,7 @@ func healthchecker(c *Conn) {
 			return
 		}
 
-		err := c.Conn.Ping(nil)
+		err := c.Conn.Ping(context.Background(), nil)
 		if err != nil {
 			if err == conn.ErrClosed {
 				// somebody else closed the connection while we were sleeping
@@ -314,7 +315,7 @@ func (s *singleRemote) PingAll(c *Client, concurrency int) {
 		return
 	}
 
-	err = cn.Conn.Ping(nil)
+	err = cn.Conn.Ping(context.Background(), nil)
 	if err != nil {
 		cn.Close()
 	}
@@ -470,7 +471,7 @@ func (g *Group) PingAll(c *Client, concurrency int) {
 			}
 
 			start := time.Now()
-			err = cn.Conn.Ping(nil)
+			err = cn.Conn.Ping(context.Background(), nil)
 			duration := time.Since(start)
 
 			if err != nil {

client/remote_test.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"crypto"
 	"crypto/x509"
 	"encoding/pem"
@@ -244,7 +245,7 @@ func NewRemoteSignerByCertFile(filepath string) (crypto.Signer, error) {
 	if err != nil {
 		return nil, err
 	}
-	s, err := c.NewRemoteSignerByPublicKey("", pub)
+	s, err := c.NewRemoteSignerByPublicKey(context.Background(), "", pub)
 	if err != nil {
 		return nil, err
 	}

cmd/gokeyless/gokeyless.go

@@ -12,6 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
+
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
@@ -48,6 +51,10 @@ type Config struct {
 	PidFile string `yaml:"pid_file" mapstructure:"pid_file"`
 
 	CurrentTime string `yaml:"current_time" mapstructure:"current_time"`
+
+	TracingEnabled    bool    `yaml:"tracing_enabled" mapstructure:"tracing_enabled"`
+	TracingAddress    string  `yaml:"tracing_address" mapstructure:"tracing_address"`
+	TracingSampleRate float64 `yaml:"tracing_sample_rate" mapstructure:"tracing_sample_rate"` // between 0 and 1
 }
 
 // PrivateKeyStoreConfig defines a key store.
@@ -106,6 +113,10 @@ func init() {
 	viper.SetDefault("metrics_port", 2406)
 	flagset.String("pid-file", "", "File to store PID of running server")
 	flagset.String("current-time", "", "Current time used for certificate validation (for testing only)")
+	flagset.Bool("tracing-enabled", false, "")
+	flagset.String("tracing-address", "", "")
+	viper.SetDefault("tracing-address", "localhost:6831")
+	flagset.Float64("tracing-sample-rate", 0, "")
 	// These override the private_key_stores value from the config file.
 	flagset.StringVar(&privateKeyDirs, "private-key-dirs", "", "Comma-separated list of directories in which private keys are stored with .key extension")
 	flagset.StringVar(&privateKeyFiles, "private-key-files", "", "Comma-separated list of private key files")
@@ -236,6 +247,22 @@ func main() {
 		fmt.Print(string(b))
 		os.Exit(0)
 	}
+	if config.TracingEnabled {
+		// jaeger failing to connect to the agent / initializing shouldn't prevent keyless from starting,
+		// so if we encounter an error we should log it but move on.
+		jaegerTransport, err := jaeger.NewUDPTransport(config.TracingAddress, 0)
+		if err != nil {
+			log.Errorf("failed to enable tracing: %s", err)
+		}
+		sampler, err := jaeger.NewProbabilisticSampler(config.TracingSampleRate)
+		if err != nil {
+			log.Errorf("failed to enable tracing: %s", err)
+		}
+		tracer, closer := jaeger.NewTracer("gokeyless", sampler, jaeger.NewRemoteReporter(jaegerTransport))
+		defer closer.Close()
+		opentracing.SetGlobalTracer(tracer)
+		log.Infof("tracing enabled: %s", sampler.String())
+	}
 
 	// If we make it here we need to ask for user input, so we need to give up
 	// and log an error instead (in case the server is running as a daemon).