Add support for target scraping with SPIFFE

config/http_config.go

@@ -32,6 +32,11 @@ import (
 	"time"
 
 	conntrack "github.com/mwitkow/go-conntrack"
+	spiffebundle "github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
+	"github.com/spiffe/go-spiffe/v2/spiffeid"
+	"github.com/spiffe/go-spiffe/v2/spiffetls"
+	spiffetlsconfig "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig"
+	spiffesvid "github.com/spiffe/go-spiffe/v2/svid/x509svid"
 	"go.yaml.in/yaml/v2"
 	"golang.org/x/net/http/httpproxy"
 	"golang.org/x/net/http2"
@@ -456,6 +461,11 @@ type DialContextFunc func(context.Context, string, string) (net.Conn, error)
 // NewTLSConfigFunc returns tls.Config.
 type NewTLSConfigFunc func(context.Context, *TLSConfig, ...TLSConfigOption) (*tls.Config, error)
 
+type SpiffeSvidAndBundleSource interface {
+	spiffesvid.Source
+	spiffebundle.Source
+}
+
 type httpClientOptions struct {
 	dialContextFunc   DialContextFunc
 	newTLSConfigFunc  NewTLSConfigFunc
@@ -465,6 +475,7 @@ type httpClientOptions struct {
 	userAgent         string
 	host              string
 	secretManager     SecretManager
+	spiffeSourceFn    func() (SpiffeSvidAndBundleSource, error)
 }
 
 // HTTPClientOption defines an option that can be applied to the HTTP client.
@@ -530,6 +541,20 @@ func WithHost(host string) HTTPClientOption {
 	})
 }
 
+// WithSpiffeSourceFactory allows SPIFFE to be used with this HTTP client.
+// The provided function should return the same X509Source on every call
+// since all clients can share the same source. The source may either
+// already exist (in which case the function can just return a fixed value)
+// or be created on demand (in which case no X509Source will be created unless
+// SPIFFE is configured and used). The returned X509Source will not be closed
+// during the lifetime of the HTTPClient. The default is that there is no
+// factory function and SPIFFE is not available.
+func WithSpiffeSourceFactory(fn func() (SpiffeSvidAndBundleSource, error)) HTTPClientOption {
+	return httpClientOptionFunc(func(opts *httpClientOptions) {
+		opts.spiffeSourceFn = fn
+	})
+}
+
 type secretManagerOption struct {
 	secretManager SecretManager
 }
@@ -584,6 +609,28 @@ func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, optFuncs ...HT
 	return NewRoundTripperFromConfigWithContext(context.Background(), cfg, name, optFuncs...)
 }
 
+func makeSpiffeDialer(configuredSpiffeID string, getSource func() (SpiffeSvidAndBundleSource, error)) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		ids := configuredSpiffeID
+		if idc := ctx.Value(SpiffeIDContextValue); idc != nil {
+			ids = idc.(string)
+		}
+		peer, err := spiffeid.FromString(ids)
+		if err != nil {
+			return nil, fmt.Errorf("unparsable SPIFFE ID %q: %w", ids, err)
+		}
+		if getSource == nil {
+			return nil, errors.New("SPIFFE requested but not configured")
+		}
+		source, err := getSource()
+		if err != nil {
+			return nil, err
+		}
+		mode := spiffetls.MTLSClientWithRawConfig(spiffetlsconfig.AuthorizeID(peer), source, source)
+		return spiffetls.DialWithMode(ctx, network, addr, mode)
+	}
+}
+
 // NewRoundTripperFromConfigWithContext returns a new HTTP RoundTripper configured for the
 // given config.HTTPClientConfig and config.HTTPClientOption.
 // The name is used as go-conntrack metric label.
@@ -607,6 +654,11 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 	}
 
 	newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
+		var dialTLS func(ctx context.Context, network, addr string) (net.Conn, error)
+		if tlsConfig == nil {
+			// Use SPIFFE
+			dialTLS = makeSpiffeDialer(cfg.TLSConfig.SpiffeID, opts.spiffeSourceFn)
+		}
 		// The only timeout we care about is the configured scrape timeout.
 		// It is applied on request. So we leave out any timings here.
 		var rt http.RoundTripper = &http.Transport{
@@ -621,6 +673,7 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 			TLSHandshakeTimeout:   10 * time.Second,
 			ExpectContinueTimeout: 1 * time.Second,
 			DialContext:           dialContext,
+			DialTLSContext:        dialTLS,
 		}
 		if opts.http2Enabled && cfg.EnableHTTP2 {
 			// HTTP/2 support is golang had many problematic cornercases where
@@ -691,6 +744,10 @@ func NewRoundTripperFromConfigWithContext(ctx context.Context, cfg HTTPClientCon
 		return rt, nil
 	}
 
+	if cfg.TLSConfig.SpiffeID != "" || cfg.TLSConfig.UseSpiffe {
+		return newRT(nil)
+	}
+
 	tlsConfig, err := opts.newTLSConfigFunc(ctx, &cfg.TLSConfig, WithSecretManager(opts.secretManager))
 	if err != nil {
 		return nil, err
@@ -1140,6 +1197,13 @@ type TLSConfig struct {
 	MinVersion TLSVersion `yaml:"min_version,omitempty" json:"min_version,omitempty"`
 	// Maximum TLS version.
 	MaxVersion TLSVersion `yaml:"max_version,omitempty" json:"max_version,omitempty"`
+	// Use SPIFFE to configure TLS. The special label `__spiffe_id__` on the
+	// scrape target configures the SPIFFE ID that must be presented.
+	UseSpiffe bool `yaml:"use_spiffe,omitempty" json:"use_spiffe,omitempty"`
+	// Use SPIFFE to configure TLS. The special label `__spiffe_id__` on the
+	// scrape target (first) or the value of this parameter (otherwise)
+	// configures the SPIFFE ID that must be presented.
+	SpiffeID string `yaml:"spiffe_id,omitempty" json:"spiffe_id,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -1181,6 +1245,10 @@ func (c *TLSConfig) Validate() error {
 		return errors.New("exactly one of cert or cert_file must be configured when a client key is configured")
 	}
 
+	if (len(c.CA) > 0 || len(c.CAFile) > 0 || len(c.CARef) > 0 || len(c.Cert) > 0 || len(c.CertFile) > 0 || len(c.CertRef) > 0 || len(c.Key) > 0 || len(c.KeyFile) > 0 || len(c.KeyRef) > 0 || len(c.ServerName) > 0 || c.InsecureSkipVerify) && (len(c.SpiffeID) > 0 || c.UseSpiffe) {
+		return errors.New("either SPIFFE settings or other TLSConfig settings may be set but not both")
+	}
+
 	return nil
 }
 
@@ -1536,3 +1604,7 @@ func (c *ProxyConfig) Proxy() (fn func(*http.Request) (*url.URL, error)) {
 func (c *ProxyConfig) GetProxyConnectHeader() http.Header {
 	return c.ProxyConnectHeader.HTTPHeader()
 }
+
+type spiffeIDContextValue bool
+
+const SpiffeIDContextValue = spiffeIDContextValue(false)

config/http_config_test.go

@@ -15,9 +15,11 @@ package config
 
 import (
 	"context"
+	"crypto"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
@@ -35,6 +37,9 @@ import (
 	"testing"
 	"time"
 
+	spiffebundle "github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
+	"github.com/spiffe/go-spiffe/v2/spiffeid"
+	spiffesvid "github.com/spiffe/go-spiffe/v2/svid/x509svid"
 	"github.com/stretchr/testify/require"
 	"go.yaml.in/yaml/v2"
 )
@@ -53,6 +58,12 @@ const (
 	MissingCert           = "missing/cert.crt"
 	MissingKey            = "missing/secret.key"
 
+	SpiffeWorkload1Cert = "testdata/spiffe.workload1.cert.pem"
+	SpiffeWorkload1Key  = "testdata/spiffe.workload1.key.pem"
+	SpiffeWorkload2Cert = "testdata/spiffe.workload2.cert.pem"
+	SpiffeWorkload2Key  = "testdata/spiffe.workload2.key.pem"
+	SpiffeBundle        = "testdata/spiffe.bundle.pem"
+
 	ExpectedMessage                   = "I'm here to serve you!!!"
 	ExpectedError                     = "expected error"
 	AuthorizationCredentials          = "theanswertothegreatquestionoflifetheuniverseandeverythingisfortytwo"
@@ -166,6 +177,37 @@ func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt
 	return testServer, nil
 }
 
+func newSpiffeTestServer() (*httptest.Server, error) {
+	handler := func(w http.ResponseWriter, _ *http.Request) {
+		fmt.Fprint(w, ExpectedMessage)
+	}
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(handler))
+
+	tlsCAChain, err := os.ReadFile(SpiffeBundle)
+	if err != nil {
+		return nil, fmt.Errorf("Can't read %s", SpiffeBundle)
+	}
+	serverCertificate, err := tls.LoadX509KeyPair(SpiffeWorkload1Cert, SpiffeWorkload1Key)
+	if err != nil {
+		return nil, fmt.Errorf("Can't load X509 key pair %s - %s", SpiffeWorkload1Cert, SpiffeWorkload1Key)
+	}
+
+	rootCAs := x509.NewCertPool()
+	rootCAs.AppendCertsFromPEM(tlsCAChain)
+
+	testServer.TLS = &tls.Config{
+		Certificates: make([]tls.Certificate, 1),
+		RootCAs:      rootCAs,
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		ClientCAs:    rootCAs,
+	}
+	testServer.TLS.Certificates[0] = serverCertificate
+
+	testServer.StartTLS()
+
+	return testServer, nil
+}
+
 func TestNewClientFromConfig(t *testing.T) {
 	newClientValidConfig := []struct {
 		clientConfig HTTPClientConfig
@@ -1275,6 +1317,131 @@ func TestTLSRoundTripper_Inline(t *testing.T) {
 	}
 }
 
+type testSpiffeSource struct{}
+
+func (*testSpiffeSource) GetX509SVID() (*spiffesvid.SVID, error) {
+	cert, err := tls.LoadX509KeyPair(SpiffeWorkload2Cert, SpiffeWorkload2Key)
+	if err != nil {
+		return nil, fmt.Errorf("Can't load X509 key pair %s - %s", SpiffeWorkload2Cert, SpiffeWorkload2Key)
+	}
+	if signer, ok := cert.PrivateKey.(crypto.Signer); ok {
+		return &spiffesvid.SVID{
+			ID:           spiffeid.RequireFromString("spiffe://example.org/workload2"),
+			Certificates: []*x509.Certificate{cert.Leaf},
+			PrivateKey:   signer,
+		}, nil
+	}
+	return nil, errors.New("private key is not crypto.Signer")
+}
+
+func (*testSpiffeSource) GetX509BundleForTrustDomain(trustDomain spiffeid.TrustDomain) (*spiffebundle.Bundle, error) {
+	if trustDomain.Name() == "example.org" {
+		bundlePem, err := os.ReadFile(SpiffeBundle)
+		if err != nil {
+			return nil, fmt.Errorf("Can't read %s", SpiffeBundle)
+		}
+		var bundle []*x509.Certificate
+		for len(bundlePem) > 0 {
+			b, more := pem.Decode(bundlePem)
+			c, err := x509.ParseCertificate(b.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Can't parse %s as a certificate", SpiffeBundle)
+			}
+			bundlePem = more
+			bundle = append(bundle, c)
+		}
+		return spiffebundle.FromX509Authorities(trustDomain, bundle), nil
+	}
+	return nil, fmt.Errorf("No bundle for trust domain %v", trustDomain)
+}
+
+func spiffeMaker() (SpiffeSvidAndBundleSource, error) {
+	return &testSpiffeSource{}, nil
+}
+
+func TestTLSRoundTripper_SPIFFE(t *testing.T) {
+	testServer, err := newSpiffeTestServer()
+	require.NoError(t, err)
+	defer testServer.Close()
+
+	testCases := []struct {
+		disabled  bool
+		useSpiffe bool
+		confID    string
+		ctxID     string
+
+		errMsg string
+	}{
+		{
+			disabled: true,
+			confID:   "spiffe://trust.domain/foo",
+			errMsg:   "SPIFFE requested but not configured",
+		},
+		{
+			confID: "spiffe://example.org/workload1",
+		},
+		{
+			confID: "spiffe://example.org/wrong",
+			errMsg: "unexpected ID",
+		},
+		{
+			confID: "unparsable",
+			errMsg: "unparsable",
+		},
+		{
+			useSpiffe: true,
+			ctxID:     "spiffe://example.org/workload1",
+		},
+		{
+			confID: "spiffe://overridden/ignored",
+			ctxID:  "spiffe://example.org/workload1",
+		},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			cfg := HTTPClientConfig{
+				TLSConfig: TLSConfig{
+					UseSpiffe: tc.useSpiffe,
+					SpiffeID:  tc.confID,
+				},
+			}
+
+			var opts []HTTPClientOption
+			if !tc.disabled {
+				opts = append(opts, WithSpiffeSourceFactory(spiffeMaker))
+			}
+			c, err := NewClientFromConfig(cfg, "test", opts...)
+			require.NoErrorf(t, err, "Error creating HTTP client: %v", err)
+			req, err := http.NewRequest(http.MethodGet, testServer.URL, nil)
+			require.NoErrorf(t, err, "Error creating HTTP request: %v", err)
+			ctx := context.Background()
+			if tc.ctxID != "" {
+				ctx = context.WithValue(ctx, SpiffeIDContextValue, tc.ctxID)
+			}
+			r, err := c.Do(req.WithContext(ctx))
+			if tc.errMsg != "" {
+				require.ErrorContainsf(t, err, tc.errMsg, "Expected error message to contain %q, got %q", tc.errMsg, err)
+				return
+			} else if err != nil {
+				t.Fatalf("Error executing HTTP request: %v", err)
+			}
+
+			b, err := io.ReadAll(r.Body)
+			r.Body.Close()
+			if err != nil {
+				t.Errorf("Can't read the server response body")
+			}
+
+			got := strings.TrimSpace(string(b))
+			if ExpectedMessage != got {
+				t.Errorf("The expected message %q differs from the obtained message %q", ExpectedMessage, got)
+			}
+		})
+	}
+}
+
 func TestTLSRoundTripperRaces(t *testing.T) {
 	bs := getCertificateBlobs(t)
 

config/testdata/spiffe.bundle.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBDCCAaqgAwIBAgIRAP6xM1DuTb3X0lAhRTLmNhMwCgYIKoZIzj0EAwIwUDEL
+MAkGA1UEBhMCVVMxDzANBgNVBAoTBlNQSUZGRTEwMC4GA1UEBRMnMzM4NTQzOTg4
+Mjg4MjIzNDk5NzE4MjE1ODYwMjk1NDUyNDcyODUxMCAXDTI1MDkwNzA5MjQzMVoY
+DzIyMjUwNzIxMDkyNDQxWjBQMQswCQYDVQQGEwJVUzEPMA0GA1UEChMGU1BJRkZF
+MTAwLgYDVQQFEyczMzg1NDM5ODgyODgyMjM0OTk3MTgyMTU4NjAyOTU0NTI0NzI4
+NTEwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQtQEBKDICx1ywQOjIiYGOO6DI8
+edAy3MxMQrWs737AgD45ixo1fNIGHfpD9v0WH3kq5C9ycB5YNpIaSdfXJiiJo2Mw
+YTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUZ5LV
+HKgEZ5Kvs50NaJboDBt+U/owHwYDVR0RBBgwFoYUc3BpZmZlOi8vZXhhbXBsZS5v
+cmcwCgYIKoZIzj0EAwIDSAAwRQIgOvUg85SrI2hmeCStTkCF9or0Vgcuzdifwfq5
+qQBJ9W8CIQD13tnaExaopel4BbweByAnXHqYEwomLlMbLs6ldz7a7w==
+-----END CERTIFICATE-----

config/testdata/spiffe.workload1.cert.pem

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICGjCCAb+gAwIBAgIQS8qrtPjeVHsY0SXsfT8B/zAKBggqhkjOPQQDAjBQMQsw
+CQYDVQQGEwJVUzEPMA0GA1UEChMGU1BJRkZFMTAwLgYDVQQFEyczMzg1NDM5ODgy
+ODgyMjM0OTk3MTgyMTU4NjAyOTU0NTI0NzI4NTEwIBcNMjUwOTA3MDkyNjUzWhgP
+MjEwMDA4MjAwOTI3MDNaMB0xCzAJBgNVBAYTAlVTMQ4wDAYDVQQKEwVTUElSRTBZ
+MBMGByqGSM49AgEGCCqGSM49AwEHA0IABO+FhNA7IYyFNA/BDOovyNVlwZVesbfj
+knv/alVHrJ8Z5PPLv5RC4EqxDUCIdYH8IcfpXbRUsxV3Y8oIRkQFWpOjgaswgagw
+DgYDVR0PAQH/BAQDAgOoMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAM
+BgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQJ/hirbRAZC32OWIHWBdVfOF67gzAfBgNV
+HSMEGDAWgBRnktUcqARnkq+znQ1olugMG35T+jApBgNVHREEIjAghh5zcGlmZmU6
+Ly9leGFtcGxlLm9yZy93b3JrbG9hZDEwCgYIKoZIzj0EAwIDSQAwRgIhAPIkKNdq
+SBb9ROIe2eo5ed1/0ay89UMjd+dxlkuyX8jAAiEAx1P1bBnzAVhuQx0YbzG2lJnK
+QBCtPcGZ/amdWZt9G0A=
+-----END CERTIFICATE-----

config/testdata/spiffe.workload1.key.pem

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgorRBVyov8trytD8G
+v094aUhNkeOlBfV6mdZfC7rm66+hRANCAATvhYTQOyGMhTQPwQzqL8jVZcGVXrG3
+45J7/2pVR6yfGeTzy7+UQuBKsQ1AiHWB/CHH6V20VLMVd2PKCEZEBVqT
+-----END PRIVATE KEY-----