diff --git a/build-index/tagclient/client.go b/build-index/tagclient/client.go index c17f2b19a..92765b553 100644 --- a/build-index/tagclient/client.go +++ b/build-index/tagclient/client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -38,6 +38,7 @@ var ( // Client wraps tagserver endpoints. type Client interface { + CheckReadiness() error Put(tag string, d core.Digest) error PutAndReplicate(tag string, d core.Digest) error Get(tag string) (core.Digest, error) @@ -70,6 +71,14 @@ func NewSingleClient(addr string, config *tls.Config) Client { return &singleClient{addr, config} } +func (c *singleClient) CheckReadiness() error { + _, err := httputil.Get( + fmt.Sprintf("http://%s/readiness", c.addr), + httputil.SendTimeout(5*time.Second), + httputil.SendTLS(c.tls)) + return err +} + func (c *singleClient) Put(tag string, d core.Digest) error { _, err := httputil.Put( fmt.Sprintf("http://%s/tags/%s/digest/%s", c.addr, url.PathEscape(tag), d.String()), @@ -311,6 +320,33 @@ func (cc *clusterClient) do(request func(c Client) error) error { return err } +// doOnce tries the request on only one randomly chosen client without any retries if it fails. +func (cc *clusterClient) doOnce(request func(c Client) error) error { + addrs := cc.hosts.Resolve().Sample(1) + if len(addrs) == 0 { + return errors.New("cluster client: no hosts could be resolved") + } + // read the only sampled addr + var addr string + for addr = range addrs { + } + err := request(NewSingleClient(addr, cc.tls)) + if httputil.IsNetworkError(err) { + cc.hosts.Failed(addr) + } + return err +} + +func (cc *clusterClient) CheckReadiness() error { + return cc.doOnce(func(c Client) error { + err := c.CheckReadiness() + if err != nil { + return fmt.Errorf("build index not ready: %v", err) + } + return nil + }) +} + func (cc *clusterClient) Put(tag string, d core.Digest) error { return cc.do(func(c Client) error { return c.Put(tag, d) }) } diff --git a/build-index/tagserver/server.go b/build-index/tagserver/server.go index 1dcafcf3e..abc6280bb 100644 --- a/build-index/tagserver/server.go +++ b/build-index/tagserver/server.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -108,6 +108,7 @@ func (s *Server) Handler() http.Handler { r.Use(middleware.LatencyTimer(s.stats)) r.Get("/health", handler.Wrap(s.healthHandler)) + r.Get("/readiness", handler.Wrap(s.readinessCheckHandler)) r.Put("/tags/{tag}/digest/{digest}", handler.Wrap(s.putTagHandler)) r.Head("/tags/{tag}", handler.Wrap(s.hasTagHandler)) @@ -145,6 +146,19 @@ func (s *Server) healthHandler(w http.ResponseWriter, r *http.Request) error { return nil } +func (s *Server) readinessCheckHandler(w http.ResponseWriter, r *http.Request) error { + err := s.backends.CheckReadiness() + if err != nil { + return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable) + } + err = s.localOriginClient.CheckReadiness() + if err != nil { + return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable) + } + fmt.Fprintln(w, "OK") + return nil +} + func (s *Server) putTagHandler(w http.ResponseWriter, r *http.Request) error { tag, err := httputil.ParseParam(r, "tag") if err != nil { diff --git a/build-index/tagserver/server_test.go b/build-index/tagserver/server_test.go index bce5ed9c5..cf2a14d89 100644 --- a/build-index/tagserver/server_test.go +++ b/build-index/tagserver/server_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -14,10 +14,12 @@ package tagserver import ( + "errors" "fmt" "io/ioutil" "net/http" "net/url" + "regexp" "strconv" "testing" "time" @@ -149,6 +151,69 @@ func TestHealth(t *testing.T) { require.Equal("OK\n", string(b)) } +func TestCheckReadiness(t *testing.T) { + for _, tc := range []struct { + name string + mockStatErr error + mockOriginErr error + expectedErrMsgPattern string + }{ + { + name: "success", + mockStatErr: nil, + mockOriginErr: nil, + expectedErrMsgPattern: "", + }, + { + name: "failure, 503 (only Stat fails)", + mockStatErr: errors.New("backend storage error"), + mockOriginErr: nil, + expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: backend for namespace 'foo-bar/\*' not ready: backend storage error`), + }, + { + name: "failure, 503 (only origin fails)", + mockStatErr: nil, + mockOriginErr: errors.New("origin error"), + expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: origin error`), + }, + { + name: "failure, 503 (both fail)", + mockStatErr: errors.New("backend storage error"), + mockOriginErr: errors.New("origin error"), + expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: backend for namespace 'foo-bar/\*' not ready: backend storage error`), + }, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + + mocks, cleanup := newServerMocks(t) + defer cleanup() + + addr, stop := testutil.StartServer(mocks.handler()) + defer stop() + + client := newClusterClient(addr) + backendClient := mockbackend.NewMockClient(mocks.ctrl) + require.NoError(mocks.backends.Register("foo-bar/*", backendClient, true)) + + mockStat := &core.BlobInfo{} + if tc.mockStatErr != nil { + mockStat = nil + } + backendClient.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat, tc.mockStatErr) + mocks.originClient.EXPECT().CheckReadiness().Return(tc.mockOriginErr).AnyTimes() + + err := client.CheckReadiness() + if tc.expectedErrMsgPattern == "" { + require.Nil(err) + } else { + r, _ := regexp.Compile(tc.expectedErrMsgPattern) + require.True(r.MatchString(err.Error())) + } + }) + } +} + func TestPut(t *testing.T) { require := require.New(t) diff --git a/mocks/build-index/tagclient/client.go b/mocks/build-index/tagclient/client.go index 4a878b7a2..7945e644e 100644 --- a/mocks/build-index/tagclient/client.go +++ b/mocks/build-index/tagclient/client.go @@ -1,160 +1,181 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/uber/kraken/build-index/tagclient (interfaces: Client) +// +// Generated by this command: +// +// mockgen -package mocktagclient . Client +// // Package mocktagclient is a generated GoMock package. package mocktagclient import ( - gomock "github.com/golang/mock/gomock" + reflect "reflect" + time "time" + tagclient "github.com/uber/kraken/build-index/tagclient" tagmodels "github.com/uber/kraken/build-index/tagmodels" core "github.com/uber/kraken/core" - reflect "reflect" - time "time" + gomock "github.com/golang/mock/gomock" ) -// MockClient is a mock of Client interface +// MockClient is a mock of Client interface. type MockClient struct { ctrl *gomock.Controller recorder *MockClientMockRecorder + isgomock struct{} } -// MockClientMockRecorder is the mock recorder for MockClient +// MockClientMockRecorder is the mock recorder for MockClient. type MockClientMockRecorder struct { mock *MockClient } -// NewMockClient creates a new mock instance +// NewMockClient creates a new mock instance. func NewMockClient(ctrl *gomock.Controller) *MockClient { mock := &MockClient{ctrl: ctrl} mock.recorder = &MockClientMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use +// EXPECT returns an object that allows the caller to indicate expected use. func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } -// DuplicatePut mocks base method -func (m *MockClient) DuplicatePut(arg0 string, arg1 core.Digest, arg2 time.Duration) error { +// CheckReadiness mocks base method. +func (m *MockClient) CheckReadiness() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckReadiness") + ret0, _ := ret[0].(error) + return ret0 +} + +// CheckReadiness indicates an expected call of CheckReadiness. +func (mr *MockClientMockRecorder) CheckReadiness() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckReadiness", reflect.TypeOf((*MockClient)(nil).CheckReadiness)) +} + +// DuplicatePut mocks base method. +func (m *MockClient) DuplicatePut(tag string, d core.Digest, delay time.Duration) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DuplicatePut", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "DuplicatePut", tag, d, delay) ret0, _ := ret[0].(error) return ret0 } -// DuplicatePut indicates an expected call of DuplicatePut -func (mr *MockClientMockRecorder) DuplicatePut(arg0, arg1, arg2 interface{}) *gomock.Call { +// DuplicatePut indicates an expected call of DuplicatePut. +func (mr *MockClientMockRecorder) DuplicatePut(tag, d, delay interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DuplicatePut", reflect.TypeOf((*MockClient)(nil).DuplicatePut), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DuplicatePut", reflect.TypeOf((*MockClient)(nil).DuplicatePut), tag, d, delay) } -// DuplicateReplicate mocks base method -func (m *MockClient) DuplicateReplicate(arg0 string, arg1 core.Digest, arg2 core.DigestList, arg3 time.Duration) error { +// DuplicateReplicate mocks base method. +func (m *MockClient) DuplicateReplicate(tag string, d core.Digest, dependencies core.DigestList, delay time.Duration) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DuplicateReplicate", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "DuplicateReplicate", tag, d, dependencies, delay) ret0, _ := ret[0].(error) return ret0 } -// DuplicateReplicate indicates an expected call of DuplicateReplicate -func (mr *MockClientMockRecorder) DuplicateReplicate(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +// DuplicateReplicate indicates an expected call of DuplicateReplicate. +func (mr *MockClientMockRecorder) DuplicateReplicate(tag, d, dependencies, delay interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DuplicateReplicate", reflect.TypeOf((*MockClient)(nil).DuplicateReplicate), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DuplicateReplicate", reflect.TypeOf((*MockClient)(nil).DuplicateReplicate), tag, d, dependencies, delay) } -// Get mocks base method -func (m *MockClient) Get(arg0 string) (core.Digest, error) { +// Get mocks base method. +func (m *MockClient) Get(tag string) (core.Digest, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0) + ret := m.ctrl.Call(m, "Get", tag) ret0, _ := ret[0].(core.Digest) ret1, _ := ret[1].(error) return ret0, ret1 } -// Get indicates an expected call of Get -func (mr *MockClientMockRecorder) Get(arg0 interface{}) *gomock.Call { +// Get indicates an expected call of Get. +func (mr *MockClientMockRecorder) Get(tag interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), tag) } -// Has mocks base method -func (m *MockClient) Has(arg0 string) (bool, error) { +// Has mocks base method. +func (m *MockClient) Has(tag string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Has", arg0) + ret := m.ctrl.Call(m, "Has", tag) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// Has indicates an expected call of Has -func (mr *MockClientMockRecorder) Has(arg0 interface{}) *gomock.Call { +// Has indicates an expected call of Has. +func (mr *MockClientMockRecorder) Has(tag interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Has", reflect.TypeOf((*MockClient)(nil).Has), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Has", reflect.TypeOf((*MockClient)(nil).Has), tag) } -// List mocks base method -func (m *MockClient) List(arg0 string) ([]string, error) { +// List mocks base method. +func (m *MockClient) List(prefix string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0) + ret := m.ctrl.Call(m, "List", prefix) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } -// List indicates an expected call of List -func (mr *MockClientMockRecorder) List(arg0 interface{}) *gomock.Call { +// List indicates an expected call of List. +func (mr *MockClientMockRecorder) List(prefix interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), prefix) } -// ListRepository mocks base method -func (m *MockClient) ListRepository(arg0 string) ([]string, error) { +// ListRepository mocks base method. +func (m *MockClient) ListRepository(repo string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRepository", arg0) + ret := m.ctrl.Call(m, "ListRepository", repo) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListRepository indicates an expected call of ListRepository -func (mr *MockClientMockRecorder) ListRepository(arg0 interface{}) *gomock.Call { +// ListRepository indicates an expected call of ListRepository. +func (mr *MockClientMockRecorder) ListRepository(repo interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepository", reflect.TypeOf((*MockClient)(nil).ListRepository), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepository", reflect.TypeOf((*MockClient)(nil).ListRepository), repo) } -// ListRepositoryWithPagination mocks base method -func (m *MockClient) ListRepositoryWithPagination(arg0 string, arg1 tagclient.ListFilter) (tagmodels.ListResponse, error) { +// ListRepositoryWithPagination mocks base method. +func (m *MockClient) ListRepositoryWithPagination(repo string, filter tagclient.ListFilter) (tagmodels.ListResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRepositoryWithPagination", arg0, arg1) + ret := m.ctrl.Call(m, "ListRepositoryWithPagination", repo, filter) ret0, _ := ret[0].(tagmodels.ListResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListRepositoryWithPagination indicates an expected call of ListRepositoryWithPagination -func (mr *MockClientMockRecorder) ListRepositoryWithPagination(arg0, arg1 interface{}) *gomock.Call { +// ListRepositoryWithPagination indicates an expected call of ListRepositoryWithPagination. +func (mr *MockClientMockRecorder) ListRepositoryWithPagination(repo, filter interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepositoryWithPagination", reflect.TypeOf((*MockClient)(nil).ListRepositoryWithPagination), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepositoryWithPagination", reflect.TypeOf((*MockClient)(nil).ListRepositoryWithPagination), repo, filter) } -// ListWithPagination mocks base method -func (m *MockClient) ListWithPagination(arg0 string, arg1 tagclient.ListFilter) (tagmodels.ListResponse, error) { +// ListWithPagination mocks base method. +func (m *MockClient) ListWithPagination(prefix string, filter tagclient.ListFilter) (tagmodels.ListResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListWithPagination", arg0, arg1) + ret := m.ctrl.Call(m, "ListWithPagination", prefix, filter) ret0, _ := ret[0].(tagmodels.ListResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListWithPagination indicates an expected call of ListWithPagination -func (mr *MockClientMockRecorder) ListWithPagination(arg0, arg1 interface{}) *gomock.Call { +// ListWithPagination indicates an expected call of ListWithPagination. +func (mr *MockClientMockRecorder) ListWithPagination(prefix, filter interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListWithPagination", reflect.TypeOf((*MockClient)(nil).ListWithPagination), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListWithPagination", reflect.TypeOf((*MockClient)(nil).ListWithPagination), prefix, filter) } -// Origin mocks base method +// Origin mocks base method. func (m *MockClient) Origin() (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Origin") @@ -163,50 +184,50 @@ func (m *MockClient) Origin() (string, error) { return ret0, ret1 } -// Origin indicates an expected call of Origin +// Origin indicates an expected call of Origin. func (mr *MockClientMockRecorder) Origin() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Origin", reflect.TypeOf((*MockClient)(nil).Origin)) } -// Put mocks base method -func (m *MockClient) Put(arg0 string, arg1 core.Digest) error { +// Put mocks base method. +func (m *MockClient) Put(tag string, d core.Digest) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Put", arg0, arg1) + ret := m.ctrl.Call(m, "Put", tag, d) ret0, _ := ret[0].(error) return ret0 } -// Put indicates an expected call of Put -func (mr *MockClientMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call { +// Put indicates an expected call of Put. +func (mr *MockClientMockRecorder) Put(tag, d interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockClient)(nil).Put), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockClient)(nil).Put), tag, d) } -// PutAndReplicate mocks base method -func (m *MockClient) PutAndReplicate(arg0 string, arg1 core.Digest) error { +// PutAndReplicate mocks base method. +func (m *MockClient) PutAndReplicate(tag string, d core.Digest) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PutAndReplicate", arg0, arg1) + ret := m.ctrl.Call(m, "PutAndReplicate", tag, d) ret0, _ := ret[0].(error) return ret0 } -// PutAndReplicate indicates an expected call of PutAndReplicate -func (mr *MockClientMockRecorder) PutAndReplicate(arg0, arg1 interface{}) *gomock.Call { +// PutAndReplicate indicates an expected call of PutAndReplicate. +func (mr *MockClientMockRecorder) PutAndReplicate(tag, d interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutAndReplicate", reflect.TypeOf((*MockClient)(nil).PutAndReplicate), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutAndReplicate", reflect.TypeOf((*MockClient)(nil).PutAndReplicate), tag, d) } -// Replicate mocks base method -func (m *MockClient) Replicate(arg0 string) error { +// Replicate mocks base method. +func (m *MockClient) Replicate(tag string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Replicate", arg0) + ret := m.ctrl.Call(m, "Replicate", tag) ret0, _ := ret[0].(error) return ret0 } -// Replicate indicates an expected call of Replicate -func (mr *MockClientMockRecorder) Replicate(arg0 interface{}) *gomock.Call { +// Replicate indicates an expected call of Replicate. +func (mr *MockClientMockRecorder) Replicate(tag interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Replicate", reflect.TypeOf((*MockClient)(nil).Replicate), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Replicate", reflect.TypeOf((*MockClient)(nil).Replicate), tag) }