Add origin readiness endpoint (#373)

As described in #371, all services should implement a "readiness" endpoint, which checks whether the service is ready to serve traffic, i.e. whether it can reach all of its dependencies.

Add readiness endpoint for origin, which checks whether all its backends are reachable.

core/digest.go

@@ -23,11 +23,6 @@ import (
 	"strings"
 )
 
-const (
-	// DigestEmptyTar is the sha256 digest of an empty tar file.
-	DigestEmptyTar = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-)
-
 // DigestList is a list of digests.
 type DigestList []Digest
 

lib/backend/manager.go

@@ -143,7 +143,7 @@ func (m *Manager) GetClient(namespace string) (Client, error) {
 	return nil, ErrNamespaceNotFound
 }
 
-// IsReady returns whether the backends are ready (reachable).
+// CheckReadiness returns whether the backends are ready (available).
 // A backend must be explicitly configured as required for readiness to be checked.
 func (m *Manager) CheckReadiness() error {
 	for _, b := range m.backends {

lib/backend/manager_test.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/uber-go/tally"
 	"github.com/uber/kraken/core"
-	"github.com/uber/kraken/lib/backend"
 	. "github.com/uber/kraken/lib/backend"
 	"github.com/uber/kraken/lib/backend/backenderrors"
 	"github.com/uber/kraken/lib/backend/namepath"
@@ -222,8 +221,8 @@ func TestManagerCheckReadiness(t *testing.T) {
 				mockStat2 = nil
 			}
 
-			c1.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat1, tc.mockStat1Err).AnyTimes()
-			c2.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat2, tc.mockStat2Err).AnyTimes()
+			c1.EXPECT().Stat(ReadinessCheckNamespace, ReadinessCheckName).Return(mockStat1, tc.mockStat1Err).AnyTimes()
+			c2.EXPECT().Stat(ReadinessCheckNamespace, ReadinessCheckName).Return(mockStat2, tc.mockStat2Err).AnyTimes()
 
 			require.NoError(m.Register(n1, c1, tc.mustReady1))
 			require.NoError(m.Register(n2, c2, tc.mustReady2))

origin/blobserver/server.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -130,6 +130,7 @@ func (s *Server) Handler() http.Handler {
 	// Public endpoints:
 
 	r.Get("/health", handler.Wrap(s.healthCheckHandler))
+	r.Get("/readiness", handler.Wrap(s.readinessCheckHandler))
 
 	r.Get("/blobs/{digest}/locations", handler.Wrap(s.getLocationsHandler))
 
@@ -179,6 +180,15 @@ func (s *Server) healthCheckHandler(w http.ResponseWriter, r *http.Request) erro
 	return nil
 }
 
+func (s *Server) readinessCheckHandler(w http.ResponseWriter, r *http.Request) error {
+	err := s.backends.CheckReadiness()
+	if err != nil {
+		return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable)
+	}
+	fmt.Fprintln(w, "OK")
+	return nil
+}
+
 // statHandler returns blob info if it exists.
 func (s *Server) statHandler(w http.ResponseWriter, r *http.Request) error {
 	checkLocal, err := strconv.ParseBool(httputil.GetQueryArg(r, "local", "false"))

origin/blobserver/server_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,13 +19,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/uber/kraken/core"
+	"github.com/uber/kraken/lib/backend"
 	"github.com/uber/kraken/lib/backend/backenderrors"
 	"github.com/uber/kraken/lib/persistedretry"
 	"github.com/uber/kraken/lib/persistedretry/writeback"
@@ -53,6 +55,60 @@ func TestHealth(t *testing.T) {
 	require.Equal("OK\n", string(b))
 }
 
+func TestReadiness(t *testing.T) {
+	for _, tc := range []struct {
+		mockStatErr    error
+		expectedErrMsg string
+		status         int
+	}{
+		{
+			mockStatErr:    nil,
+			expectedErrMsg: "",
+			status:         http.StatusOK,
+		},
+		{
+			mockStatErr:    backenderrors.ErrBlobNotFound,
+			expectedErrMsg: "",
+			status:         http.StatusOK,
+		},
+		{
+			mockStatErr:    errors.New("failed due to backend error"),
+			expectedErrMsg: fmt.Sprintf("not ready to serve traffic: backend for namespace '%s' not ready: failed due to backend error", backend.ReadinessCheckNamespace),
+			status:         http.StatusServiceUnavailable,
+		},
+	} {
+		require := require.New(t)
+
+		cp := newTestClientProvider()
+
+		s := newTestServer(t, master1, hashRingMaxReplica(), cp)
+		defer s.cleanup()
+
+		backendClient := s.backendClient(backend.ReadinessCheckNamespace, true)
+		mockStat := &core.BlobInfo{}
+		if tc.mockStatErr != nil {
+			mockStat = nil
+		}
+
+		backendClient.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat, tc.mockStatErr)
+
+		resp, err := httputil.Get(
+			fmt.Sprintf("http://%s/readiness", s.addr))
+
+		if tc.status == http.StatusOK {
+			defer resp.Body.Close()
+			require.Equal(tc.status, resp.StatusCode)
+			require.NoError(err)
+			b, _ := ioutil.ReadAll(resp.Body)
+			require.Equal("OK\n", string(b))
+		} else {
+			require.True(httputil.IsStatus(err, tc.status))
+			require.True(strings.Contains(err.Error(), tc.expectedErrMsg))
+			require.Nil(resp)
+		}
+	}
+}
+
 func TestStatHandlerLocalNotFound(t *testing.T) {
 	require := require.New(t)
 
@@ -117,7 +173,7 @@ func TestStatHandlerNotFound(t *testing.T) {
 	d := core.DigestFixture()
 	namespace := core.TagFixture()
 
-	backendClient := s.backendClient(namespace)
+	backendClient := s.backendClient(namespace, false)
 
 	backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound)
 
@@ -192,7 +248,7 @@ func TestDownloadBlobNotFound(t *testing.T) {
 	d := core.DigestFixture()
 	namespace := core.TagFixture()
 
-	backendClient := s.backendClient(namespace)
+	backendClient := s.backendClient(namespace, false)
 	backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound)
 
 	err := cp.Provide(master1).DownloadBlob(namespace, d, ioutil.Discard)
@@ -280,7 +336,7 @@ func TestGetMetaInfoDownloadsBlobAndReplicates(t *testing.T) {
 
 	blob := computeBlobForHosts(ring, s1.host, s2.host)
 
-	backendClient := s1.backendClient(namespace)
+	backendClient := s1.backendClient(namespace, false)
 	backendClient.EXPECT().Stat(namespace,
 		blob.Digest.Hex()).Return(core.NewBlobInfo(int64(len(blob.Content))), nil).AnyTimes()
 	backendClient.EXPECT().Download(namespace, blob.Digest.Hex(), mockutil.MatchWriter(blob.Content)).Return(nil)
@@ -317,7 +373,7 @@ func TestGetMetaInfoBlobNotFound(t *testing.T) {
 	d := core.DigestFixture()
 	namespace := core.TagFixture()
 
-	backendClient := s.backendClient(namespace)
+	backendClient := s.backendClient(namespace, false)
 	backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound)
 
 	mi, err := cp.Provide(master1).GetMetaInfo(namespace, d)
@@ -582,7 +638,7 @@ func TestReplicateToRemoteWhenBlobInStorageBackend(t *testing.T) {
 	blob := core.NewBlobFixture()
 	namespace := core.TagFixture()
 
-	backendClient := s.backendClient(namespace)
+	backendClient := s.backendClient(namespace, false)
 	backendClient.EXPECT().Stat(namespace,
 		blob.Digest.Hex()).Return(core.NewBlobInfo(int64(len(blob.Content))), nil).AnyTimes()
 	backendClient.EXPECT().Download(namespace, blob.Digest.Hex(), mockutil.MatchWriter(blob.Content)).Return(nil)

origin/blobserver/testutils_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -158,9 +158,9 @@ func newTestServer(
 	}
 }
 
-func (s *testServer) backendClient(namespace string) *mockbackend.MockClient {
+func (s *testServer) backendClient(namespace string, mustReady bool) *mockbackend.MockClient {
 	client := mockbackend.NewMockClient(s.ctrl)
-	if err := s.backendManager.Register(namespace, client, false); err != nil {
+	if err := s.backendManager.Register(namespace, client, mustReady); err != nil {
 		panic(err)
 	}
 	return client