From ab251474d759c775f8bb8c2ca7e0f03d9496189a Mon Sep 17 00:00:00 2001 From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:43:10 +0000 Subject: [PATCH 1/5] feat: Add HealthChecker interface and default impl --- gateway/internal/router/health_checker.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 gateway/internal/router/health_checker.go diff --git a/gateway/internal/router/health_checker.go b/gateway/internal/router/health_checker.go new file mode 100644 index 0000000..8cb1f28 --- /dev/null +++ b/gateway/internal/router/health_checker.go @@ -0,0 +1,17 @@ +package router + +import ( + "log" +) + +type HealthChecker interface { + IsHealthy(providerName string) bool +} + +type DefaultHealthChecker struct{} + +func (d *DefaultHealthChecker) IsHealthy(providerName string) bool { + // Placeholder for actual health check logic + // Currently returns true, assuming all providers are healthy + return true +} From d37b54bf0bc3b9e4d336d3c99bc0fc4fbf13ac02 Mon Sep 17 00:00:00 2001 From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:44:40 +0000 Subject: [PATCH 2/5] feat: Updated gateway/internal/router/round_robin. --- gateway/internal/router/round_robin.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/gateway/internal/router/round_robin.go b/gateway/internal/router/round_robin.go index 60ac24c..5d88108 100644 --- a/gateway/internal/router/round_robin.go +++ b/gateway/internal/router/round_robin.go @@ -1,7 +1,9 @@ package router import ( + "log" "sync/atomic" + "gateway/internal/router" // Importing to use HealthChecker ) const ( @@ -26,9 +28,22 @@ func (r *RoundRobinRouter) Iterator() RouterIterator { func (r *RoundRobinRouter) Next() *RouterConfig { providerLen := len(r.providers) - // Todo: make a check for healthy provider - idx := r.idx.Add(1) - 1 - model := &r.providers[idx%uint64(providerLen)] + // Iterate through providers to find a healthy one + var healthyProvider *RouterConfig + originalIdx := r.idx.Load() + for i := 0; i < providerLen; i++ { + idx := (originalIdx + uint64(i)) % uint64(providerLen) + if router.DefaultHealthChecker{}.IsHealthy(r.providers[idx].Name) { + healthyProvider = &r.providers[idx] + r.idx.Add(1) + break + } + } + + if healthyProvider == nil { + log.Println("Error: No healthy providers available.") + return nil + } - return model + return healthyProvider } From 4ddece299009bf57563dbc2e19f6eafa66465ffc Mon Sep 17 00:00:00 2001 From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:46:32 +0000 Subject: [PATCH 3/5] feat: Updated gateway/internal/router/priority.go --- gateway/internal/router/priority.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/gateway/internal/router/priority.go b/gateway/internal/router/priority.go index 7b0f80e..71a5136 100644 --- a/gateway/internal/router/priority.go +++ b/gateway/internal/router/priority.go @@ -2,6 +2,8 @@ package router import ( "sync/atomic" + "log" + "gateway/internal/router" // Importing to use HealthChecker ) const ( @@ -21,11 +23,20 @@ func NewPriorityRouter(providers []RouterConfig) *PriorityRouter { } func (r *PriorityRouter) Next() (*RouterConfig, error) { - idx := int(r.idx.Load()) - - // Todo: make a check for healthy provider - model := &r.providers[idx] - r.idx.Add(1) - - return model, nil + providerLen := len(r.providers) + originalIdx := r.idx.Load() + var healthyProvider *RouterConfig + for i := 0; i < providerLen; i++ { + idx := (originalIdx + uint64(i)) % uint64(providerLen) + if router.DefaultHealthChecker{}.IsHealthy(r.providers[idx].Name) { + healthyProvider = &r.providers[idx] + r.idx.Store(idx + 1) + break + } + } + if healthyProvider == nil { + log.Println("Error: No healthy providers available.") + return nil, fmt.Errorf("no healthy providers available") + } + return healthyProvider, nil } From 81cf9967ab6c4eab4920a3b9e234ac7224248c04 Mon Sep 17 00:00:00 2001 From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:47:17 +0000 Subject: [PATCH 4/5] feat: Updated gateway/internal/api/v1/models.go --- gateway/internal/api/v1/models.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gateway/internal/api/v1/models.go b/gateway/internal/api/v1/models.go index 160f991..df24e33 100644 --- a/gateway/internal/api/v1/models.go +++ b/gateway/internal/api/v1/models.go @@ -13,6 +13,11 @@ func (s *V1Handler) ListModels(ctx context.Context, req *connect.Request[llmv1.M allProviderModels := map[string]*llmv1.ProviderModels{} for name := range base.ProviderRegistry { + // Check if the provider is healthy before fetching models + if !router.DefaultHealthChecker{}.IsHealthy(name) { + continue + } + provider, err := s.iProviderService.GetProvider(provider.Provider{Name: name}) if err != nil { continue From 1601da062828c257bd4cbd26ecaaa0f1a37b0bda Mon Sep 17 00:00:00 2001 From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:55:53 +0000 Subject: [PATCH 5/5] feat: Updated gateway/internal/api/v1/providers.go --- gateway/internal/api/v1/providers.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/gateway/internal/api/v1/providers.go b/gateway/internal/api/v1/providers.go index 54d5ddd..b65e4c5 100644 --- a/gateway/internal/api/v1/providers.go +++ b/gateway/internal/api/v1/providers.go @@ -20,6 +20,10 @@ func (s *V1Handler) ListProviders(ctx context.Context, req *connect.Request[empt data := []*llmv1.Provider{} for _, provider := range providers { + // Check if the provider is healthy before adding to the list + if !router.DefaultHealthChecker{}.IsHealthy(provider.Info().Name) { + continue + } providerInfo := provider.Info() data = append(data, &llmv1.Provider{ Title: providerInfo.Title, @@ -34,6 +38,11 @@ func (s *V1Handler) ListProviders(ctx context.Context, req *connect.Request[empt } func (s *V1Handler) GetProvider(ctx context.Context, req *connect.Request[llmv1.GetProviderRequest]) (*connect.Response[llmv1.GetProviderResponse], error) { + // First, check if the provider is healthy + if !router.DefaultHealthChecker{}.IsHealthy(req.Msg.Name) { + return nil, errors.NewNotFound("Provider is unhealthy") + } + provider, err := s.iProviderService.GetProvider(provider.Provider{Name: req.Msg.Name}) if err != nil { return nil, errors.NewNotFound(err.Error()) @@ -63,6 +72,11 @@ func (s *V1Handler) GetProvider(ctx context.Context, req *connect.Request[llmv1. } func (s *V1Handler) CreateProvider(ctx context.Context, req *connect.Request[llmv1.CreateProviderRequest]) (*connect.Response[llmv1.CreateProviderResponse], error) { + // First, check if the provider is healthy + if !router.DefaultHealthChecker{}.IsHealthy(req.Msg.Name) { + return nil, errors.NewNotFound("Provider is unhealthy") + } + provider := provider.Provider{Name: req.Msg.Name, Config: req.Msg.Config.AsMap()} p, err := s.iProviderService.GetProvider(provider) @@ -111,6 +125,11 @@ func (s *V1Handler) CreateProvider(ctx context.Context, req *connect.Request[llm } func (s *V1Handler) UpsertProvider(ctx context.Context, req *connect.Request[llmv1.UpdateProviderRequest]) (*connect.Response[llmv1.UpdateProviderResponse], error) { + // First, check if the provider is healthy + if !router.DefaultHealthChecker{}.IsHealthy(req.Msg.Name) { + return nil, errors.NewNotFound("Provider is unhealthy") + } + provider := provider.Provider{Name: req.Msg.Name, Config: req.Msg.Config.AsMap()} p, err := s.iProviderService.GetProvider(provider) @@ -172,6 +191,11 @@ func (s *V1Handler) UpsertProvider(ctx context.Context, req *connect.Request[llm } func (s *V1Handler) GetProviderConfig(ctx context.Context, req *connect.Request[llmv1.GetProviderConfigRequest]) (*connect.Response[llmv1.GetProviderConfigResponse], error) { + // First, check if the provider is healthy + if !router.DefaultHealthChecker{}.IsHealthy(req.Msg.Name) { + return nil, errors.NewNotFound("Provider is unhealthy") + } + p, err := s.iProviderService.GetProvider(provider.Provider{Name: req.Msg.Name}) if err != nil { return nil, errors.NewNotFound(err.Error())