From c81d40a22c99001405af6d786856d18b52468213 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Tue, 25 Nov 2025 16:59:57 +0000 Subject: [PATCH 1/7] initial online batch support Signed-off-by: Shimi Bandiel --- cmd/batch/main.go | 16 +++ cmd/batch/runner/runner.go | 99 +++++++++++++ pkg/batch/README.md | 61 ++++++++ pkg/batch/api.go | 41 ++++++ pkg/batch/random_robin_policy.go | 47 ++++++ pkg/batch/random_robin_policy_test.go | 41 ++++++ pkg/batch/redis/redisimpl.go | 200 ++++++++++++++++++++++++++ pkg/batch/worker.go | 131 +++++++++++++++++ pkg/batch/worker_test.go | 144 +++++++++++++++++++ pkg/metrics/metrics.go | 6 +- test/integration/redisimpl_test.go | 56 ++++++++ 11 files changed, 841 insertions(+), 1 deletion(-) create mode 100644 cmd/batch/main.go create mode 100644 cmd/batch/runner/runner.go create mode 100644 pkg/batch/README.md create mode 100644 pkg/batch/api.go create mode 100644 pkg/batch/random_robin_policy.go create mode 100644 pkg/batch/random_robin_policy_test.go create mode 100644 pkg/batch/redis/redisimpl.go create mode 100644 pkg/batch/worker.go create mode 100644 pkg/batch/worker_test.go create mode 100644 test/integration/redisimpl_test.go diff --git a/cmd/batch/main.go b/cmd/batch/main.go new file mode 100644 index 000000000..2d4d0359d --- /dev/null +++ b/cmd/batch/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "os" + + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/llm-d/llm-d-inference-scheduler/cmd/batch/runner" +) + +func main() { + + if err := runner.NewRunner().Run(ctrl.SetupSignalHandler()); err != nil { + os.Exit(1) + } +} diff --git a/cmd/batch/runner/runner.go b/cmd/batch/runner/runner.go new file mode 100644 index 000000000..83dde7318 --- /dev/null +++ b/cmd/batch/runner/runner.go @@ -0,0 +1,99 @@ +package runner + +import ( + "context" + "flag" + "net/http" + + "github.com/llm-d/llm-d-inference-scheduler/pkg/batch" + "github.com/llm-d/llm-d-inference-scheduler/pkg/batch/redis" + uberzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" +) + +type Runner struct { +} + +var ( + setupLog = ctrl.Log.WithName("setup") + logVerbosity = flag.Int("v", logging.DEFAULT, "number for the log level verbosity") + concurrency = flag.Int("concurrency", 8, "number of concurrent workers") + endpoint = flag.String("endpoint", "", "inference endpoint") +) + +func NewRunner() *Runner { + return &Runner{} +} + +func (r *Runner) Run(ctx context.Context) error { + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + initLogging(&opts) + + /*if *tracing { + err := common.InitTracing(ctx, setupLog) + if err != nil { + return err + } + }*/ + + ////////setupLog.Info("GIE build", "commit-sha", version.CommitSHA, "build-ref", version.BuildRef) + + // Validate flags + if err := validateFlags(); err != nil { + setupLog.Error(err, "Failed to validate flags") + return err + } + + // Print all flag values + flags := make(map[string]any) + flag.VisitAll(func(f *flag.Flag) { + flags[f.Name] = f.Value + }) + setupLog.Info("Flags processed", "flags", flags) + + httpClient := &http.Client{ + // TODO: configure + } + var policy batch.RequestPolicy = batch.NewRandomRobinPolicy() + + var impl batch.Flow = redis.NewRedisMQFlow("localhost:6379") + requestChannel := policy.MergeRequestChannels(impl.RequestChannels()).Channel + for w := 1; w <= *concurrency; w++ { + go batch.Worker(ctx, *endpoint, httpClient, requestChannel, impl.RetryChannel(), impl.ResultChannel()) + } + + impl.Start(ctx) + + return nil +} + +// TODO: is this dup of +func initLogging(opts *zap.Options) { + // Unless -zap-log-level is explicitly set, use -v + useV := true + flag.Visit(func(f *flag.Flag) { + if f.Name == "zap-log-level" { + useV = false + } + }) + if useV { + // See https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/log/zap#Options.Level + lvl := -1 * (*logVerbosity) + opts.Level = uberzap.NewAtomicLevelAt(zapcore.Level(int8(lvl))) + } + + logger := zap.New(zap.UseFlagOptions(opts), zap.RawZapOpts(uberzap.AddCaller())) + ctrl.SetLogger(logger) +} + +func validateFlags() error { + + return nil +} diff --git a/pkg/batch/README.md b/pkg/batch/README.md new file mode 100644 index 000000000..19e21c0ac --- /dev/null +++ b/pkg/batch/README.md @@ -0,0 +1,61 @@ +# Batch Processor + +## Overview +The batch processor (BP) provides asynchronous workflows for variable SLO-based inference requests. + + +## Architecture + +An underlying implementation should provide persistent messaging that adhere to the interface defined in [api.go](api.go). + +A pluggable request policy is used to merge multiple request channels into a single request channel on which the batch worker is listening. + +An example for such a policy is a [Random Robin Policy](random_robin_policy.go). + +Each [Batch Processor worker](worker.go) is responsible for pulling requests from the merged request channel, submit to the IGW and apply retry logic if needed. + + + +### Requests + +Request messages should have the following format: +```json +{ + "id" : "unique identifier for result mapping", + "deadline" : "deadline in Unix seconds", + "payload" : {regular inference payload} +} +``` + +Example: +```json +{ + "id" : "19933123533434", + "deadline" : "1764045130", + "payload": {"model":"food-review","prompt":"hi", "max_tokens":10,"temperature":0} +} +``` + +### Results + +Messages on the results channel will have the following structure: + +```json +{ + "id" : "id mapped to the request", + "payload" : {/*inference payload*/} , + // or + "error" : "error's reason" +} +``` + + +## Implementations + +### Redis + +An example implementation based on Redis is provided which behaves as follows: + +- Redis Lists as the request queues. +- Redis Sorted Set as the retry exponential backoff implementation. +- Redis List as the result queue. diff --git a/pkg/batch/api.go b/pkg/batch/api.go new file mode 100644 index 000000000..1e88fc79d --- /dev/null +++ b/pkg/batch/api.go @@ -0,0 +1,41 @@ +package batch + +import "context" + +type Flow interface { + // starts processing requests. + Start(ctx context.Context) + + // returns the channel for requests. Implementation is responsible for populating this channel. + RequestChannels() []RequestChannel + // returns the channel that accepts messages to be retries with their backoff delay. + RetryChannel() chan RetryMessage + // returns the channel for storing the results. + ResultChannel() chan ResultMessage +} + +type RequestPolicy interface { + MergeRequestChannels(channels []RequestChannel) RequestChannel +} + +type RequestMessage struct { + Id string `json:"id"` + RetryCount int `json:"retry_count,omitempty"` + DeadlineUnixSec string `json:"deadline"` + Payload map[string]any `json:"payload"` +} + +type RequestChannel struct { + Channel chan RequestMessage + Metadata map[string]any +} + +type RetryMessage struct { + RequestMessage + BackoffDurationSeconds float64 +} + +type ResultMessage struct { + Id string `json:"id"` + Payload map[string]any `json:"payload"` +} diff --git a/pkg/batch/random_robin_policy.go b/pkg/batch/random_robin_policy.go new file mode 100644 index 000000000..e2497ac74 --- /dev/null +++ b/pkg/batch/random_robin_policy.go @@ -0,0 +1,47 @@ +package batch + +import "reflect" + +func NewRandomRobinPolicy() RequestPolicy { + return &RandomRobinPolicy{} +} + +type RandomRobinPolicy struct { +} + +func (r *RandomRobinPolicy) MergeRequestChannels(channels []RequestChannel) RequestChannel { + mergedChannel := make(chan RequestMessage) + + cases := make([]reflect.SelectCase, len(channels)) + for i, ch := range channels { + cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch.Channel)} + } + + go func() { + for { + i1, val, ok := reflect.Select(cases) + if !ok { + // one of the channels is closed, remove it + newCases := make([]reflect.SelectCase, 0, len(cases)-1) + for i2, c := range cases { + if i2 != i1 { + newCases = append(newCases, c) + } + } + cases = newCases + if len(cases) == 0 { + close(mergedChannel) + break + } + } else { + mergedChannel <- val.Interface().(RequestMessage) + } + + } + }() + + return RequestChannel{ + Channel: mergedChannel, + Metadata: map[string]any{}, + } +} diff --git a/pkg/batch/random_robin_policy_test.go b/pkg/batch/random_robin_policy_test.go new file mode 100644 index 000000000..3964eac61 --- /dev/null +++ b/pkg/batch/random_robin_policy_test.go @@ -0,0 +1,41 @@ +package batch + +import ( + "testing" +) + +func TestProcessAllChannels(t *testing.T) { + msgsPerChannel := 5 + channels := []RequestChannel{ + {Channel: make(chan RequestMessage, msgsPerChannel), Metadata: map[string]any{}}, + {Channel: make(chan RequestMessage, msgsPerChannel), Metadata: map[string]any{}}, + {Channel: make(chan RequestMessage, msgsPerChannel), Metadata: map[string]any{}}, + } + policy := NewRandomRobinPolicy() + + // Send messages to each channel + for i, ch := range channels { + for range msgsPerChannel { + ch.Channel <- RequestMessage{Id: string(rune('A' + i))} + } + } + mergedChannel := policy.MergeRequestChannels(channels).Channel + close(channels[0].Channel) + close(channels[1].Channel) + close(channels[2].Channel) + + counts := map[string]int{} + totalMessages := msgsPerChannel * 3 + for range totalMessages { + msg := <-mergedChannel + counts[msg.Id]++ + + } + + for i := range 3 { + id := string(rune('A' + i)) + if counts[id] != msgsPerChannel { + t.Errorf("Expected %d messages from channel %s, got %d", msgsPerChannel, id, counts[id]) + } + } +} diff --git a/pkg/batch/redis/redisimpl.go b/pkg/batch/redis/redisimpl.go new file mode 100644 index 000000000..19a21753b --- /dev/null +++ b/pkg/batch/redis/redisimpl.go @@ -0,0 +1,200 @@ +package redis + +import ( + "context" + "encoding/json" + "fmt" + + "strconv" + "time" + + "github.com/llm-d/llm-d-inference-scheduler/pkg/batch" + "github.com/redis/go-redis/v9" + + "sigs.k8s.io/controller-runtime/pkg/log" + logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" +) + +var ( + // TODO: externalize + requestQueueName = "batch-queue" + retryQueueName = "batch-sortedset-retry" + resultQueueName = "batch-queue-result" +) + +// TODO: think about what to do if Redis is down +type RedisMQFlow struct { + rdb *redis.Client + requestChannel chan batch.RequestMessage + retryChannel chan batch.RetryMessage + resultChannel chan batch.ResultMessage +} + +func NewRedisMQFlow(addr string) *RedisMQFlow { + rdb := redis.NewClient(&redis.Options{ + Addr: addr, + + // TODO: check specific version of go-redis. might require higher version. + // Explicitly disable maintenance notifications + // This prevents the client from sending CLIENT MAINT_NOTIFICATIONS ON + // import "github.com/redis/go-redis/v9/maintnotifications" + // MaintNotificationsConfig: &maintnotifications.Config{ + // Mode: maintnotifications.ModeDisabled, + // }, + }) + return &RedisMQFlow{ + rdb: rdb, + requestChannel: make(chan batch.RequestMessage), + retryChannel: make(chan batch.RetryMessage), + resultChannel: make(chan batch.ResultMessage), + } +} + +func (r *RedisMQFlow) Start(ctx context.Context) { + go requestWorker(ctx, r.rdb, r.requestChannel, requestQueueName) + + go addMsgToRetryWorker(ctx, r.rdb, r.retryChannel, retryQueueName) + + go retryWorker(ctx, r.rdb, r.requestChannel) + + go resultWorker(ctx, r.rdb, r.resultChannel, resultQueueName) +} +func (r *RedisMQFlow) RequestChannels() []batch.RequestChannel { + return []batch.RequestChannel{{Channel: r.requestChannel, Metadata: map[string]any{}}} +} + +func (r *RedisMQFlow) RetryChannel() chan batch.RetryMessage { + return r.retryChannel +} + +func (r *RedisMQFlow) ResultChannel() chan batch.ResultMessage { + return r.resultChannel +} + +// Listening on the results channel and responsible for writing results into Redis. +func resultWorker(ctx context.Context, rdb *redis.Client, resultChannel chan batch.ResultMessage, resultsQueueName string) { + for { + select { + case <-ctx.Done(): + return + + case msg := <-resultChannel: + bytes, err := json.Marshal(msg) + var msgStr string + if err != nil { + msgStr = fmt.Sprintf(`{"id" : "%s", "error": "%s"}`, msg.Id, "Failed to marshal result to string") + } else { + msgStr = string(bytes) + } + err = publishRedis(ctx, rdb, resultsQueueName, msgStr) + if err != nil { + // TODO: ??? + + } + } + } +} + +// pulls from Redis Queue and put in the request channel +func requestWorker(ctx context.Context, rdb *redis.Client, msgChannel chan batch.RequestMessage, queueName string) { + sub := rdb.Subscribe(ctx, queueName) + defer sub.Close() + + // redis.WithChannelSize(100) -- TODO: consider exposing to config + ch := sub.Channel() + for { + select { + case <-ctx.Done(): + return + + case rmsg := <-ch: + var msg batch.RequestMessage + + err := json.Unmarshal([]byte(rmsg.Payload), &msg) + if err != nil { + // TODO: log failed to unmarshal message. + fmt.Println(err) + continue // skip this message + + } + msgChannel <- msg + } + } + +} + +// Puts msgs from the retry channel into a Redis sorted-set with a duration Score. +func addMsgToRetryWorker(ctx context.Context, rdb *redis.Client, retryChannel chan batch.RetryMessage, sortedSetName string) error { + for { + select { + case <-ctx.Done(): + return nil + + case msg := <-retryChannel: + score := float64(time.Now().Unix()) + msg.BackoffDurationSeconds + bytes, err := json.Marshal(msg.RequestMessage) + if err != nil { + fmt.Printf("Failed to marshal message for retry in Redis: %s", err.Error()) + continue // skip this message. TODO: log + } + err = rdb.ZAdd(ctx, sortedSetName, redis.Z{ + Score: score, + Member: string(bytes), + }).Err() + + if err != nil { + fmt.Printf("Failed to add message for retry in Redis: %s", err.Error()) + // TODO: + } + } + } + +} + +// TODO +// Every second polls the sorted set and publishes the messages that need to be retried into the request queue +func retryWorker(ctx context.Context, rdb *redis.Client, msgChannel chan batch.RequestMessage) { + for { + select { + case <-ctx.Done(): + return + + default: + currentTimeSec := float64(time.Now().Unix()) + results, err := rdb.ZRangeByScore(ctx, retryQueueName, &redis.ZRangeBy{ + Min: "0", + Max: strconv.FormatFloat(currentTimeSec, 'f', -1, 64), + }).Result() + if err != nil { + panic(err) + } + for _, msg := range results { + var message batch.RequestMessage + err := json.Unmarshal([]byte(msg), &message) + if err != nil { + fmt.Println(err) + + } + err = rdb.ZRem(ctx, retryQueueName, msg).Err() + if err != nil { + fmt.Println(err) + + } + // TODO: Publish to request channel or directly to request queue in Redis? + msgChannel <- message + } + time.Sleep(time.Second) + } + } + +} + +func publishRedis(ctx context.Context, rdb *redis.Client, channelId, msg string) error { + logger := log.FromContext(ctx) + err := rdb.Publish(ctx, channelId, msg).Err() + if err != nil { + logger.V(logutil.DEFAULT).Error(err, "Error publishing message:%s\n", err.Error()) + return err + } + return nil +} diff --git a/pkg/batch/worker.go b/pkg/batch/worker.go new file mode 100644 index 000000000..54d650a4e --- /dev/null +++ b/pkg/batch/worker.go @@ -0,0 +1,131 @@ +package batch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "math/rand" + "net/http" + "strconv" + "time" + + "github.com/llm-d/llm-d-inference-scheduler/pkg/metrics" + "sigs.k8s.io/controller-runtime/pkg/log" + logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" +) + +var baseDelaySeconds = 2 + +func Worker(ctx context.Context, endpoint string, httpClient *http.Client, requestChannel chan RequestMessage, + retryChannel chan RetryMessage, resultChannel chan ResultMessage) { + + logger := log.FromContext(ctx) + for { + select { + case <-ctx.Done(): + logger.V(logutil.DEFAULT).Info("Worker finishing.") + return + case msg := <-requestChannel: + payloadBytes := parseAndValidateRequest(resultChannel, msg) + if payloadBytes == nil { + continue + } + + sendInferenceRequest := func() { + logger.V(logutil.DEBUG).Info("Sending inference request.") + result, err := httpClient.Post(endpoint, "application/json", bytes.NewBuffer(payloadBytes)) + if err != nil { + resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to send request to inference: %s", err.Error())) + return + } + defer result.Body.Close() + // Retrying on any server-side error. Assuming shedding is included here. + if result.StatusCode >= 500 && result.StatusCode < 600 { + retryMessage(msg, retryChannel, resultChannel) + } else { + payloadBytes, err := io.ReadAll(result.Body) + if err != nil { + // Retrying on IO-read error as well. + retryMessage(msg, retryChannel, resultChannel) + } else { + var resultPayload map[string]any + err := json.Unmarshal(payloadBytes, &resultPayload) + if err != nil { + // Not retrying on unmarshalling error. + resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to unmarshal inference result payload: %v", err)) + return + } + resultChannel <- ResultMessage{ + Id: msg.Id, + Payload: resultPayload, + } + } + } + } + sendInferenceRequest() + } + } +} +func parseAndValidateRequest(resultChannel chan ResultMessage, msg RequestMessage) []byte { + deadline, err := strconv.ParseInt(msg.DeadlineUnixSec, 10, 64) + if err != nil { + resultChannel <- CreateErrorResultMessage(msg.Id, "Failed to parse deadline, should be in Unix seconds.") + return nil + } + + if deadline < time.Now().Unix() { + resultChannel <- CreateDeadlineExceededResultMessage(msg.Id) + return nil + } + + payloadBytes, err := json.Marshal(msg.Payload) + if err != nil { + resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to marshal message's payload: %s", err.Error())) + return nil + } + return payloadBytes +} + +// If it is not after deadline, just publish again. +func retryMessage(msg RequestMessage, retryChannel chan RetryMessage, resultChannel chan ResultMessage) { + deadline, err := strconv.ParseInt(msg.DeadlineUnixSec, 10, 64) + if err != nil { // Can't really happen because this was already parsed in the past. But we don't care to have this branch. + resultChannel <- CreateErrorResultMessage(msg.Id, "Failed to parse deadline. Should be in Unix time") + return + } + secondsToDeadline := deadline - time.Now().Unix() + if secondsToDeadline < 0 { + resultChannel <- CreateDeadlineExceededResultMessage(msg.Id) + } else { + msg.RetryCount++ + backoffDurationSeconds := math.Min( + float64(baseDelaySeconds)*(math.Pow(2, float64(msg.RetryCount))), + float64(secondsToDeadline)) + + jitter := rand.Float64() - 0.5 + finalDuration := backoffDurationSeconds + jitter + if finalDuration < 0 { + finalDuration = 0 + } + metrics.Retries.Inc() + retryChannel <- RetryMessage{ + RequestMessage: msg, + BackoffDurationSeconds: finalDuration, + } + + } + +} +func CreateErrorResultMessage(id string, errMsg string) ResultMessage { + return ResultMessage{ + Id: id, + Payload: map[string]any{"error": errMsg}, + } +} + +func CreateDeadlineExceededResultMessage(id string) ResultMessage { + return CreateErrorResultMessage(id, "deadline exceeded") +} diff --git a/pkg/batch/worker_test.go b/pkg/batch/worker_test.go new file mode 100644 index 000000000..fd6769bae --- /dev/null +++ b/pkg/batch/worker_test.go @@ -0,0 +1,144 @@ +package batch + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" +) + +func TestRetryMessage_deadlinePassed(t *testing.T) { + retryChannel := make(chan RetryMessage, 1) + resultChannel := make(chan ResultMessage, 1) + msg := RequestMessage{ + Id: "123", + RetryCount: 0, + DeadlineUnixSec: fmt.Sprintf("%d", time.Now().Add(time.Second*-10).Unix()), + } + retryMessage(msg, retryChannel, resultChannel) + if len(retryChannel) > 0 { + t.Errorf("Message that its deadline passed should not be retried. Got a message in the retry channel") + return + } + if len(resultChannel) != 1 { + t.Errorf("Expected one message in the result channel") + return + + } + result := <-resultChannel + if result.Payload["error"] != "deadline exceeded" { + t.Errorf("Expected error to be: 'deadline exceeded', got: %s", result.Payload["error"]) + } + +} + +func TestRetryMessage_retry(t *testing.T) { + retryChannel := make(chan RetryMessage, 1) + resultChannel := make(chan ResultMessage, 1) + msg := RequestMessage{ + Id: "123", + RetryCount: 0, + DeadlineUnixSec: fmt.Sprintf("%d", time.Now().Add(time.Second*10).Unix()), + } + retryMessage(msg, retryChannel, resultChannel) + if len(resultChannel) > 0 { + t.Errorf("Should not have any messages in the result channel") + return + } + if len(retryChannel) != 1 { + t.Errorf("Expected one message in the retry channel") + return + } + retryMsg := <-retryChannel + if retryMsg.RetryCount != 1 { + t.Errorf("Expected retry count to be 1, got %d", msg.RetryCount) + } + +} + +// RoundTripFunc is a type that implements http.RoundTripper +type RoundTripFunc func(req *http.Request) (*http.Response, error) + +// RoundTrip executes a single HTTP transaction, obtaining the Response for a given Request. +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +// NewTestClient returns an *http.Client with its Transport replaced by a custom RoundTripper. +func NewTestClient(fn RoundTripFunc) *http.Client { + return &http.Client{ + Transport: RoundTripFunc(fn), + } +} + +func TestSheddedRequest(t *testing.T) { + msgId := "123" + httpclient := NewTestClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: nil, + Header: make(http.Header), + }, nil + }) + requestChannel := make(chan RequestMessage, 1) + retryChannel := make(chan RetryMessage, 1) + resultChannel := make(chan ResultMessage, 1) + ctx := context.Background() + + go Worker(ctx, "http://localhost:30080/v1/completions", httpclient, requestChannel, retryChannel, resultChannel) + deadline := time.Now().Add(time.Second * 100).Unix() + + requestChannel <- RequestMessage{ + Id: msgId, + RetryCount: 0, + DeadlineUnixSec: fmt.Sprintf(("%d"), deadline), + Payload: map[string]any{"model": "food-review", "prompt": "hi", "max_tokens": 10, "temperature": 0}, + } + + select { + case r := <-retryChannel: + if r.Id != msgId { + t.Errorf("Expected retry message id to be %s, got %s", msgId, r.Id) + } + case <-resultChannel: + t.Errorf("Should not get result from a 5xx response") + + } + +} +func TestSuccessfulRequest(t *testing.T) { + msgId := "123" + httpclient := NewTestClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: nil, + Header: make(http.Header), + }, nil + }) + requestChannel := make(chan RequestMessage, 1) + retryChannel := make(chan RetryMessage, 1) + resultChannel := make(chan ResultMessage, 1) + ctx := context.Background() + + go Worker(ctx, "http://localhost:30080/v1/completions", httpclient, requestChannel, retryChannel, resultChannel) + + deadline := time.Now().Add(time.Second * 100).Unix() + + requestChannel <- RequestMessage{ + Id: msgId, + RetryCount: 0, + DeadlineUnixSec: fmt.Sprintf(("%d"), deadline), + Payload: map[string]any{"model": "food-review", "prompt": "hi", "max_tokens": 10, "temperature": 0}, + } + + select { + case <-retryChannel: + t.Errorf("Should not get a retry from a 200 response") + case r := <-resultChannel: + if r.Id != msgId { + t.Errorf("Expected result message id to be %s, got %s", msgId, r.Id) + } + } + +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 03aadebdb..1fe25beda 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,12 +27,16 @@ var ( }, []string{"decision_type"}, // "decode-only" or "prefill-decode" ) + Retries = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_request_retries_total", + Help: "Total number of batch request retries.", + }) ) // GetCollectors returns all custom collectors for the llm-d-inference-scheduler. func GetCollectors() []prometheus.Collector { return []prometheus.Collector{ - SchedulerPDDecisionCount, + SchedulerPDDecisionCount, Retries, } } diff --git a/test/integration/redisimpl_test.go b/test/integration/redisimpl_test.go new file mode 100644 index 000000000..da21d8c8c --- /dev/null +++ b/test/integration/redisimpl_test.go @@ -0,0 +1,56 @@ +package integration_test + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/llm-d/llm-d-inference-scheduler/pkg/batch" + "github.com/llm-d/llm-d-inference-scheduler/pkg/batch/redis" +) + +const ( + redisURL = "localhost:6379" +) + +func TestRedisImpl(t *testing.T) { + ctx := context.Background() + flow := redis.NewRedisMQFlow(redisURL) + flow.Start(ctx) + + flow.RetryChannel() <- batch.RetryMessage{ + RequestMessage: batch.RequestMessage{ + Id: "test-id", + DeadlineUnixSec: strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10), + Payload: map[string]any{"model": "food-review", "prompt": "hi", "max_tokens": 10, "temperature": 0}, + }, + BackoffDurationSeconds: 2, + } + totalReqCount := 0 + for _, value := range flow.RequestChannels() { + totalReqCount += len(value.Channel) + } + + if totalReqCount > 0 { + t.Errorf("Expected no messages in request channels yet") + return + } + if len(flow.ResultChannel()) > 0 { + t.Errorf("Expected no messages in result channel yet") + return + } + time.Sleep(3 * time.Second) + + mergedChannel := batch.NewRandomRobinPolicy().MergeRequestChannels(flow.RequestChannels()) + + select { + case req := <-mergedChannel.Channel: + if req.Id != "test-id" { + t.Errorf("Expected message id to be test-id, got %s", req.Id) + } + case <-time.After(2 * time.Second): + t.Errorf("Expected message in request channel after backoff") + } + +} From 55a52c1bda25ccc116d314f0a8278fc26d85fb87 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Mon, 1 Dec 2025 11:57:00 -0800 Subject: [PATCH 2/7] Add files via upload upload image Signed-off-by: Shimi Bandiel --- .../batch_processor_redis_architecture.png | Bin 0 -> 54426 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/images/batch_processor_redis_architecture.png diff --git a/docs/images/batch_processor_redis_architecture.png b/docs/images/batch_processor_redis_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..7b96766f998018f7c21988e15d8f1f56388783b9 GIT binary patch literal 54426 zcmeFZc{r5s-#1Quq)6I^l4vJpQX)H*q)4*QjGbhe2HA}pqU>uix?b%Zzh*pXc>nU$6J&<%9d$0(>HTTwGiN zckkRXxRvwKXhFhnmio^yYZKmDlrc?9=SkeA72{s~daYev$1(ppLlwdi9( z1oxxKxxBf}t)iLw_eGV}IW;-&5h<7$Phh|Q_w)a!8*s|KpqIdYxb@F-;{bT2^STT# zaJsaQJZ_jx-<$dB)sz@v4d9#Lfp=0z_vU?_gzHNh8(EryU;+Gs(qL|~C zTqDEitITrRvz}hK#AA)XCHcIS_(y>tFLT>wn|QTVUHn^?#A-RPHrInD!_ZwY_f4<9 z4%;j8bB*6i$D{QE6`<)19oxZt#0r0y_%$g-RxK%Hca-8aD6$pbG#=P%sR4fHy*ZP* zDO!Q-y#z%hZ|2QEX**)n#7vZT6Wc%Zm}I;zKYk%>wk73>ZT{<@8Ym*444ShaUcyB3 z4UZ#+rzE1^#MJz%(?B82{i{~*$GJ>Vboa~A4da8_Etu7CT*ySL`9H=Em&Q8=om}A)_xYK(vaE0;QV0@K7Z&}k60sLQ;7hfqb&tnXXRazenccRYD}eQY z6~FI-s@34&mHSLB@_eU+*q}CaU642fx3LB zPkn8mG_U2XxAr`Y-N~`t2Z-&*_>N~Gwo*gTt>A%P%6faOLWt)j;+44XSJla><*oNM zpPk2ANP)Aw11fEU_^{LMSAqh5y>twX-uf09B;ba)^rY*fUI>Ghkhq4)nwwO6^rZ8S|pr%0g1NL{83cm-|TJWq5MaAE~ysPVPxuU(T{5~2gT9X`Y@Gs zO=n=5?Tz=OpTl1Sk@Lu7px$}jgFdWD~9#QKV<=W3%p>!j%|u9k}PVP|8pn=f18B z=z{njHX1xtw(Gvwc%-;K-o?nuj}bZYGqRXJ?6>Fj%T6=1A<1ELM?^70Q;ll+_w!7@zFx$~C6s}w_4qOABBd}fkJ7k*|6BEt; zZmRSi;QZ;jma}a?S(dyQNr#afusNQUK?K!t^q+M>QVn_a!1j`F{PiG-=B@R!Dk_@ z8^ep$=UjIdJbnUtsTA$%Kj@L~a9&rO3+R{f8;`5LFW_ z4RtL?JNL`@itPy3-R1F)8H(eB&9>G&xe>ri+J`MpN~NjvvF!X}nf!3ehp;n2I|n)5 z+2c{+0Lb%O0NV=I<0DTYs|Hw*S6`Kh?QxH%(1L%?F#=xgh#5=99>QKE3*$wq3z0A1 z704mt7`(};v;w7uj{Md-upm^O@cWGlCy(d4egTX;?&?3VSU^8Oio#+;C6uiHnRoCX zUaY5I)=jiX-0Zn&rlCmeF<7kM7FsUC2O9{9R<>TEIvyXJ09Axy%S30`E4SN}tVM+n zgowGYetP6&2Q)s}9xf6ZVGgR2@-bP;|7&un(9RDdm&DQ`>62Iaw58RwDIWEwFkP(E z9Xo4_2xyr`!yNp8eM=CiNzm5ZtoJC6eewL%j%Q` z9$2>H=sxdB+p3?7ne!swz)#Im8jV_Jy1~$VK{x zQ7K*hrKirtlb^JYoa$R&5ReYu9v$K0%~^s*}Q) z{Ijjr4`ijGQRTIdSYJI}7i@{uh)@dcU`_GzL$Ib4;ag3K@;_k=L|=KB#?ELj%Osl5 z9ceJaAU_e3Si`eQAx8_9bv`pi?T5H$hZYIl<~MhKDh=_<<|*DeYW{uleBOE?U6`au z@+sP^E0%hP_kK&boG`=2-X0imvaN`F%jb$jW!_XG1nVKIHTgcE7EY>KAMwxuf z5a+~Uu3%Kx8@t)If5dHH-072UR_6kfQW2_yWeynv+k<(gXVB5VC9s0Y;`eJ& zzh5<9?OS|Bmj>L)fj<*kWzXqn7-pdqv4-Vg?PHt?4>G%BP+?59JkKPW#~ta*!nz33 zX)Fx;0UK!16iL|tTf<$;!HG`WYH=z%hklO1ZwQJt`=mv)_BF`6ZX#GVv8w*~_)!Ed zbrYM1ox*bedz*IE@F>DPPt4y1z)dC5P`U3Z6+>fe%AH;%TGP-267(yyp0MAyIjCq` zaH)=d+0U}gES?-WFF$EBA|yZo2$ovvB#ZD3wy@&w%3RElFN$e28>ARI#?5ooDz^Qm%lrTCsy-Q4obP;1Zkz8RHus&mTW|{jnilum zn%>r%w6D&7rD0bQ(DDc?0Z)p%qd>r=^UDkQIooPyBItf4_dma0=7^FO=6pjEVv)^L zSn;OkC_|VaWqJrM3>r-6DyXVUh$*P5PhgtwPZ%svsz(jo_;Z0X=Q^arM`Ix4jycGg zbOvde1$k8wXOQQ-{0KKz*pr0MFE{nQB!rtNG)cx+mz(-uYIf=7Obg~KDW&s~;(^B* z16JeLmpnb&TM!r^Pc&Qq>!LxkGq_QlJ|LBkFD_nL<>5@|e(w#*ga8gJ*?O-|vs2an zGS%Pc;@|yP2YXzkTd+g5Lwtq8s1iY?DBdL-+UPy0F zlccib-C%Y7>;L*flgH;tdDu!ws`;hT#|Kq1mz1IROVD)o8RL-gR;~0gR4mgoUZxCu z$Zh-S8fSulIBivqwy*kH=KTBB;v^@?nPj!V*P0U}Zx5b#KDK(!jA}cyeazeK%^q2F zEmAZV@K)vtLMg@F=nJl!ZAXz)Zdns#Uz%gFNX57Qvvi=@O)qWVZri9fXp# z4RK1uQT-}s^|V{vnFT49fX+;sL>HqlJ-dhAUnAV}!r&cd&ko5`ExdG9h;OhHM-Wi& za=)D^hn?q&CKjSRG53onR$QWrC)TcA040l{*!lFc4By!Cu}E!~GCjJ4e{`RhFW4IVF<(iezGrJgdw-><&P zH^?@!W`* zF5ACR36_4kb)fo=`LR}SjP6+^(%5z2#~pJKXT(|LlYclMSuNJY1wwqB`@U6ysb%@WZ}A1SQvERXiDrA zk#{(rN;%Ph3`DKcHz~qS)ME)A&$0FxWkUIVh4mTc&nPlp~YNGt@o2 z)?0#9rzmXOcgeXC4ttd*G1#>OyvgD&Lw!Ca=*RvI@^d8!&iG&^{|(Fkl42e2ZIh+2 zQd}UCJu1{8$u;N&*dueoy&XIIaVPSov-~e5vaqwGqIHGIyGE=d*cx`f#OsBELd1En zuo5k#NG;U-OX9X?r0G6sX=Ip zd;O>=rCxj0E3C=(ehT2{Ou4T2z`*Yf9mWW_!|_`EAC8z?|9=6GrRG)s z@C~y~Khg~+=2>Bma3J)m=iMaifOk85a-UUyX}QAX7?X(wpJw8_FYTC#g(h=Lr6%zH`YOb1N3k? zP>M_vubEZ23{Qv;{{FPCpfJQ**i!rRWHW9BMe}A)0iG4-u08Jvd28z#ye`;!;+m^hPC4ktX`h+#Gj z%C~JNygEgLhjlM)^juRpz*=Rxwd#SuamPR~4y5*&I7q!=YTY%7XxVd88FJ*6>YqZJ zwHA$!1BlZ`wDoI!$i9WD)&zy#XErkzWbSc)3hEo@C+qEsPmzk9NsTqnJuxvYFWWSYsQUpVeKpZC1Np9ZyR?w|_QsYs zsVJrMA-Cc8D%8-Il2rwNRcZeP*73$N%j!KGz_;i2ghRvpwRPX}N3CJ!iea2XI=yKn zh_K1IMs2zv@wlYAO)n=vO0U1AIXvrPUJs`gUot*j)#kOV_9n{sw{n#3ddSjr-L%F$ z`gf@DAA|2O0Hx(zr#FQr`H4lnn>T(grA931%H*ee44MB3ccb=5>FK#{|Ehb% zg+tkh)sgR0^Epv5eoI#$57uwcV*WkHovuf%vNeVtnla10-G%Aqkf@s&J`3LiFPT!h2c1qXj0yvLex z%xjTx@Surv9&f}M6R%iVso>S2tvBelu{!^8!7ixy<)ihCRgHC&`{#aPUI@HmKcWB0YWH}&Y}G3Nf0ap4 zb@5a8pG)+~8S=>7oe^8p}xxaxT4^&t`eoOjRzgNoBby*G%19?a(5}yZsP1 zf544y|2E##@)A`+dONhfok(lFl~UF@4r(CM>;^}YNNztJi$~9XX5})ho)~S`0*gV; z;c1O$n+Sgd)#6d;?F)4;rWSm)>Uv?2^A87hUlGGVH*3FM3XTSt0ya}+YKZh~^91rR z*(gPo8N;puByOwyW6)<_JDsp$h|In>x4|*N;EX$a5z3D%?ABLEmd}e2(pX{*f1-?%>IR2n(y|1AkSSpy;mEcg) z)gHL0#O~{QW`05SEo$g^QOc~L1*pAnL#Sq&SvV=Y6xBDk`^Q`n0_k5r;xrKY2#}{{NDA+UEGGAU%JYZ ztSlqEWKpG$Ns;B@)PTV*SU^2KR0}nB)^>3K?I6w?-+MEi+AIX7FdJQ|O-zv*t`zHT zm6OP>tOs121R(cOUPfkY$1YR&Wd3DewN1N)>2O*LIpEK*Ow+ZEMD(jrjutT;V~VF% zcvJP%d?BjY1P=`jh*caK^TXbVrk7k`_cu!WV^)-c>5|p?A~Rr7E~gUX4uh%hYlt6f zOp@XM^lMmA1z5bsWjjJ~0}mxzVy5`wOq+~uM}R|B_hJlPM0fQWLYTAB^m!96V^rB@ zLfIy}>}Z`?)l5Y>{R~jD%|O&VS~x4%yajGpN1tMCL`k*AW~@p*sq-mInn#~j{n{zL zQ!{w$f={AVYP1BBwD^rtT!B zE|;eyY=$a7mCXrg@pTPo{}qKsYq8|R+tGp`iNdU^T~!mcvY2BiaA;EAlz_AE7nTYd z5vlB0muT51i-;9?eUTY=;pmjF`pV6}V&?`kRM!?&mc_n`X|YFU zmBO}~IOJv6+g7cttT%oB)yQz1h@O>4-DnNFISbo9aR4&4nqXwG(a$8Xp-w)p~>MnNL8j zV$*-glG*0=JsW#~{AqH4&EimgoV3R#d^kLGmXq(@^IvB!%;a*J8VvakFNc84rjunB zQ~j3fY=#A{ku{2_+mtbQp{K~syhTuDqI-e%Q78`(DJTD0(0Hm~qUn`Qep5(sLY}<&Y z#@RnR?uiiyh(58H5WBo*|9--tLo|Dq!xXAZfWkXUaRi!tAA0a)Q*k&zfI9&f@)lODjKyXK@DpkW;Uuurp=9t9St&C zBx|83(q7Y@NWEBI$jsMmQ|W$Z+)*mo@3C-+HD*%xbb8cCu$b@GNuxj6Ev9ah0KF$b ztY=Tu*he2)F1cOzRcgX#IAE(!$$-1?GrV~yTVeA&rO3`mtu?eoWG&%#GRmrNx5%8Y z1iUFoP(w?ylCn<}ur*%+^5yw>z7)(x6WUzP*sRIZSEO2BT_8}86LkEnm+#ZIe$Uw2 z=COV#uW&eY*J?&7dfvrrF0PWAh)TMj9M2!FJc9r=0F_!mVO@HgzQK*^-qZsEL@UqP z&Bi=oi{teefs?&gma7ZQRFxH`ahHBBG)3>j!mBm^dK+`;Z%u}y1B|%}`g)0>Y9l{@ z5cfk|22JZ($Bo<0dlqp~0%LAFolHCUc ziKg6z>D{5TkxaH+RxQx6(6nmpi8XmqAOWRm&(~0!?iT;#YqJD8Mj9z9zpr zZh0VsYSQ|rhnGv_nx$V5&J}0-!$9Asl;1``xOYRo*HnD(WZ(M$F~QVGl#iLY=PbKw z&Bl(dUXUZE8R?wYVr4Cp%M}GA9|7sCfezkd37NPRpx!tcU?K35vhY~obaC)5#m>(j zX>@y1Ikt7Il@q`7@8(ZJyF~-OhBg|WR1T|q+VS~b0Q=8~JID@n&%hvCDHp!~Ze0%0 zC?g!+Ii@30J?Kd{`Fzu~MaYy(8lI)2nV#A6!uP~Vj=Q4U=nqmPGr=`zq2IlzG|aNV zz4uBhs|O+V+tNf%5MEa|%>uL^lh9P?nK@R{$5WDV(_+3R?Bom~EWIG!f@W55Nt=;( ztY_6w5bXt)!e%z-3uSXr#;i~p1R8RFfq+Ww2`Nk;?qJ)U>}r#RFV{Td8@c02+QbS% zjKGO4a89;i913`&+?PV=h%i3sc&eA9AGqpr+(eIyYhgx;{j!MfEcL7fNJ-hz07%xq zRscxB=$L*r&Q4GI7~;GWWG;&*M~ESx_hJ?(mu#;bpZ!o^XWoV@@ipm4n&5Oh5kVb% zYG!=GU!^R57aZj&K@Be-OE5oqa?q)(waNZ_Xr7!ZT&)w8g1R*Qp_lnmXbm0<&C{V2gxQTEp6?kZ>Cz=!QF7(K5&mt9Wezjj5OA;#kV8(#49*1s1gzxr zXwQl&ryQ8J5I8}RB1XGUUZCTn_QsWbLHEjDxxU0+@ zn-QvM=(d1^egl6|<`4LgnY#)%=bDmtzU1kV!sKMlgC}6HAx6$;^J0x@Z?lR!X5IG5 zuc$J($wF2w3K@!AQ<^H}i=L97f{tc5NWEG98@k%-?$+8$ai1rB9g3qx-0f(CN2jwI zzB7zcC+l=9is_x*W$_MJI*8=6!`#M6{`buA_I{{oEc9{{My}c2vBnF^vtsylkG~MkCe%Jr8$fNHoSTo2x zbj1rp-A>F4JM$YlA{u($6%sEXHIE`UZcRsF!t@faZ4iISC*Bx@euO_V4^tw(lLJ&J zxNxf7`NNii|Aa)|SmwJ-9JY*|Q0+A^5W;Zw*USA^W>{%phWUmNgZ zauBd3ynTqPbm99cP{E0_P0`1ebqh>!Ig{1*ta42rAxRrpH*gDlU?&ra)rnu3jsLqj zY1Fo!`tXKf-Uk4(N#h^}KrNi=Lek+BrzLwoO7t<>4RX-Kg3QN@G4yH z*ZM!Y>pNCxI7!L(PSjTzbXoA_z(FQS#&#*F*^o4|d6SS?-$9jc(+ETTk}yrg{e`9I zpnLeneHNFZQs7C&q>y!(G3tBnG_>Y}Ou^4jhH`h&JLMxxJU-7YvBD^x+p_UoCFon(DarF^M9-yXRB-858pUV37 zVTQe&J#(-3#F%O6D?HjwiPU+T!s!K1lTv-uf7ZX`bKVH4OaL;kNR;f-#?FqlI^B|n z0WCQ;N^PANcNFm5$%j=cz(|KWF7?kIMsqVbh`1urM8BAQ-0872+_i%3RMp<8v@`fH z1i|X>lGv+-tGI!!kz4df{?zYlj(I+h{r0tthQRBB0=g7K16)1SL!lS|B40N-?r=Ot71~&4!G)mHgP`zp zC-@%Q`nR!I{Q%F@{>k`nK9}j~!e^gME8lX^6e;XP3dY_!;c@N**ri~F(%N^bisDjl z^xZMgrGD(SKab>O0e5X`C?XP=Z0~^N+)7666#9v&0I-8Nvg(4rJ9l?euGWtUDB!5l z|3JpsXJBx!ZseooS9s0_-_Akmy_^?qLVtL0qjMa#el==c{w7OWe4g!FyYF~=her`p zZKYC?n0y!*9Q!FQU0Wp*fwyD#K0SYHCXB+7MYdWL3(*Xom&qBVHf&nu@>H!jh zLf<`ru9;Nxl{*!uRpi{vnV*)oeshzQ|4iKWE~i)hy1}{Bgx967bi@iT!yv(UfcX~I zT1%!wfaIj-2=ZGLg}gcx*u4y?W%f8iC2IH@{M&F{CI~L z0noWfo&^H-`}dIZlzxvyX0u6Nm{q3Y(w6IUha&o2QUeq*+jq?X^PuPuEN8 zH*JxO%HJZ+ENc?HYW(fo%Ryj~>&=~KbVQf}8?ij7X%8W|J(Gm+7?;y7?$w^~WVW#kNlabYW(sn&f6NzWRC4ythS9+1B0?Jewj zn0XhtcvDQlwHR(PakHW-4?_l z^Pr^43c4ogsIW`~uvl=!N3ju|;rkK@9(OZ_aMq8*5W*7BnPx(~b)rl=bm1}yOp;?T z7(y|q1m6l(SO4bLK|yF_u5-cU8UA_n@D1kCn$7e{k{4;^&%_>QN0u8&536s9=2ivOP9MiIAsW^G z=KjumE7XTBW>H-%^$oZ=oTc$U8;1(Do=81;Z%^SH@$~pEwLCHG#2P3A`WX7NM!zQ* z6{)4-8~VwACQ$wDEdTnots+)k=!EiCV3;CNWa&}Wc%mXvksPqKQzQqI4(?em0AiUrZ84wjKvI5r?O{BJl$Hv^CwK;~Cd`epMMofqTjXO|TS;3d z7kq`c+;173!{9~7+33-LBwDp|!86YTz8EJw>OJRHf1cQOQru;xT!>9Kk6b6yJU`_I zmTh6LGI8>dRMP=++YDxqW!t$7c^*>{h%E{cOcLJ|oti=0*6Pj?>Z1v6sP6#t{>+SK zEqymNPKCqQeXF&?x+SIhLhqIt1HR7@$7SiP3@!59WE4O1dlGpuHX8L9{{SoWb6`-O zy;V%%Zr{oduw*lW!kN+5cQiA+=TO)fla{ZfXiM~;gLY8?SC)&A{n^+lL65)VM z&a&;9KJ>`MiSwjhU4_J75PW2T`3e#Mudp+vkx5h$92xWCbZr%n4aDrMGU znC2k)Zh5S)Wv#X!gb{vAo8NYJ8vVAgsyRb!BoiX29JJxo0Rfgg6;q-8TjG1j@%Zdr zer3{|DY(~K%%{aZ7^}&~qESsmhazU4y_~z%CPFRruUW?DFg;)MJKw385Yv`}kE{&! z!nH6FOwFk_cZ|FjGTRGUbdqy$(T&E!o1Y9HW^~+97y2KJdO@6=a1kj2hmqgv zX|R-vpQWPHLT&|DTc4P@6^E^nPBALABM3&FqBa{ruTbGyk%$cv%YOn`?)rq&-iolB zW)X!)8wwPy9~w)yN^ksN#eG&eKkz0{B!c~k?nydSWE*L+`~)L7qi2hWYMZ`WOa3^ppwlfW7rAi# z!80b$0Qur!_wR@m8p8KFM*z9??LQZcUwDHB6^VTxR=8v=HvA2o?<`p-rAibEYHy}G z%slgfqNl4w(i;{ty3B}TZ%l`@K&feT z$?e-ei*tLt7fYGzk7f{o8m&gA-0I!3oh&OC;OuF?Dhvxv<*cstzqtXE3^q|kzfpzS zoBCpM3nrok?^UFGIH?zh$%3VBBNL-$Uee0$TEek29pkcOU(kYAP?zKp)CC8O6(Z zU6~4bm&oT@ApEx&o99YJKMZVJO%|8pc(vGWF>K2_-m8-QJPY#J%ott-G=Sv9h?Ic1 zcdJs9_Q7#b@x@$*h_`t^ONpuhB|pK~7Bu5uGn+C^9%{>3LxBz|{_GjBG(9SH2JPnl z5Xo50{Ak80HMFvAUj>mi5*>gOTR-{jsjMN=x%P^w7La&hKxFVCDS&J}F0yFedrfTt zrA)J5&!q1n$(B;qI@(B+#wZBDV-b`F(8oKE00 z$?n~SDI;JGK@v(Acl~49hV#(`r^3FPSDzaYX#%ba+0FM2UI%1&{G-yzsasLi%Ow_W ztnVIg5q~eQJ`^m9m}%j61<>b8ww=;Xmcc(;vuw;@#w^x&zu=EyPXCxt;slpOfJ@m& zfYb6$rhVn%r%+hWZfiQ@n@#NxM5fq%c%%3+MH}+)#4*^x8?!#ZORR1L!Otm-;CX{?)^_EM+I%bzC&hrp{50Nsf$ad^CV zYC6k5RUc0Ee%%$2QPzyDFk3film6sRR7s|PgVZPMa`Y5?!5h}ZIz1coZ8*w_o8a`q$BUeX z=FsMs#iv0==^sGw=Z@t=y&^#R!n3fS5$=+)Y3$|VU@H)*PG+&Xv$Om&;epVA;UDg# zEhVBjZh|YwekvZJ*_12U!#A=PgbHEu%#JBjSt%?^)!_7@O>xrcs=;w^@kD);Q%WU< zfS|iZd;alUY78g*=H$h8atel_H#|BL>>h9%)2Ku+fG~$x@Y`qwy$5i$o+zQ~3A5~j zbiE;9j2t)69;MzNzoQB5DLiE^o!u9yfH>co_Vcvw5;ahbLiqk;`YRe(OWp&1eN=Y&gAW%8<(?(?vY72vHL zOT^Rj$iXE7{U#%PkZSL2=pL0axVME@Jo4r|qo2IGNP9E|bn!kEIhm%#TDtMw2%BCt zo};26^J!tX@+_!u7c4+6a$?hkB)zd8PmfE9#BjQ08_;Hk698;5;dGVk?I!#x~4rn0_brY~Qds={eC--jT{gA(u`_}k`KCd}AH!rIB5M@MSCqZi z^tuu>?cZae54xIDJvOh4J7dZZajwrV!haS-VS(^cIMC~bxssjRYpB*GqF$HDY`!|J zpQ#}ya*Ka&#G^lzI{0%JGEP z1-c&GfP|Us2i|9kHEs^wJl0dv9Pf$ToxF9{xH)W~f+qq+-$NfLH{_bxXwUyFnF>m# z39e|GRiFN*twK~Bd`q6!QfY0Kejz6ZkCfEv{@Ae(eoMEv&@}3!_9xqlU6s`J;^g`7 zkgwnPAlU5mnT#`LYPB7dyBUzg?%g<=+8SK^OP(VU%8UU_>OrINtNi#!PY73E0&HDJ znuhwa_7uM>HA7RRIyV&A-L<065dP5Vb0J|yVu0`bfUlXBmpx$k_8SSmZp4O8pz`;c zrgvYZ8XtwLL76q)uWt{|EO~DxbeF*T$t*z|%C}+Dj;xuL=^>=tE7Uq(dp@4LMeN0D z31rjGcYJmf(J7{ZIvyaFH(d^?Zw$%RX*T^f`+)}}<2t?ZR71+Ygd50>=S^OQiC>r! z>}BBw61xEoNNC~ZL}&(EZa+NoqBFF@RM>RXSf^KoSnYmGS`hKJ@Mg?VE$d^3;A^=; zk>#A!;uUs}ODoDcQ?Nlqhv?YdRUkOLOgDiDu#NSbFQ)5iuafj!f{=Jn1r|7zSL z2@V!E1J0R{Q)oNjWC1@mqs|Ce$cPKl9d#$sAc-nlff#R0=;X}-4aMmzlp%M>`)u3A zpGVvjL7znkg;f;&nr2#bt>YYRJ$Xy*XsaN#jg=<^-^Zyy_fAE3qz;eC9M1HwYaST* zc!keq<85J`U1qaa`(J_=hLojkF%?!;#9)hBFK?zKI~{4&@3n)%`|j3Zr7$sCvLd&| z&p1Wl)hGQZt>#D4(krG{W5}y}Y2)m+`vnG^=yYYeM_JOp}QA?}S#G zs-uz?i|8f{vjC1`i_lSPnBGU`Ok+Sz?~2{JkSDRC$OzX;onf^crIt|nXmon{X8 zI*G#T{9iEn-`2VI@8DZ#i7b}Zoh*8qK zgQ?89GFvlJV$~$4&DEyV3FHKN3I|n}#?^39DXdwe2OqAL*V_C4=#=OPo6fMZS>97% z2CZvCI{{!$@SZNIZv(=gID^$&WtII8g0u=Y#E>ki$5VU!giDTpV}f|0|L5sE$0*Kg z3Dd|}hkHP6FX_w8e&ewE>xOaGCF~z^8|`F*ljd@VP6zx7Tv{8!_0U<|?KVgRXs=%^rE|4VTREfmBG)|tjO#;OITs~EdB8}hN0-0$ zH6A?*?XNPO)U3^cq{0sXO|wIsvhsL$+v44w!Rivi#>A(<+Y1Z`;@rT4p=CVsjn3lLJi|p~i;iccz{O&zi?c~(?V%tBsZCEqc zL1Y&$y?uN9M7WFCWfzhx);vk(MFr}zOH(z#Qv(gUNYlh^T-W98M4R)K!mox!yP(n< z1zb002ik!*dZhQR>Q$tpoSy?P9huc>P&%)1Aj@EQ2J+2PcsfDLWj|di7{6M)!gU+j zwT#JSHf0o7fI+~Gn3Ft;>E@)DQE{IE(|4g9Bjo~M&_y$D)d3Iq;wcwX36dUJFnYs2 zL!bxgO-d#Q?e&Oo_`WSn{;&^LIEI&$Wx=!XkVN<%lIuTX{?XiuP?+qb>22O4OJ(7? z$tB>EoWkllzSHNlh6*va)_I^r~s zsapvu^th0DK(%=hp8boBnIbiFIs_g}ZC3^!Qrg^wJoVS~#r_d`IcoGwJ98zM7n*t3c&{~b zN@)A2*&R~tYvcC|ZpQ28@sG$u#Lo%t639{u!TXl19DMXPYaojjBdnw$2X7h=*xBK( zoCS1!*Y@xYi0=i#U6D63_sN}GQ6Z(1qh+tpb`V^I6^T%+`oXE=9MJW;*0F8y=4SS# zVEG{C zs1rZ4c&kwN)jJyz25J9VDcf!ya??%^VV|h&0HI{7%XT`iRQ3AX;&r%y@3?QrTJ1Y z{VHjzoZZ6GD#;MDO}iq1N4p+(n|kM@IolELEs~yr7<qZLE3YmNG6{6GZ7jY7h9p*mG;;W*DBkt$`e6yEz z(}YziZB^X2kE}7FVfjbwh~!6gY)+DOHoE-Q36~fGQso=P|F^&#+=FbJ*Vt_%75aAoTo(GZocl!4c36Edtr2Ot@b!TRLSE4t?T`9 zt|#J2N%K6L@zSpPxCzR}M8x|0>v>LcU#hU;;1(s?k-bhm3mahIJ&Ur(+oam{i$k;Q zl3K&Q&#aVz((hqs{@O2cy?Igxi2;|37R1|&;aW;6nZ3<&4DAVEo3vXiC92r<>b`=D z1&qiP_$gOMQVuo%JPas;rC!ojAn<~rJn6vpQI0k(l4ZVSu$;D%outEtV|clq2Q;V9 z0pR?|$)=1e(R{Y8CwuEVuO@9Z0G*)S*_P|)O}^Zcojs#kpd_05_}u z1QX|z)=nVH#Afr9|I5(I%PjMXpXMhW7al%b;nJh*o%F^f5~F<~$obJdeNqUyjYPXZ z{pY!yg;l^O#AGeOFyr7JuI`F%arASrj-05n|EvE!J{GlZ{EvnW>`KR;7lbYpb@8pk zr)+<~#)1dltc67XBpG!P`NUA0A<8kTL~IP&jgL9ayLCU=+Ux>xQ_nBejJocak4yJw z)0Kd?DI(+U8foa(O-RbrF%i>jtyWSwhF^TV>I01tJw+%Y{i@26>zM!cOQv;y*X2ng zpg%XTn%R`~G)oWC233E8vGz3u@p5&?0+K$o0y@T^n8qAdHnQgYv)$3Ecq<;OYVkVtAH($BjWq6_3uwPG8Yq^ktk6SgUxEeB+R)Vz6K=%C zP2#g79fAAqh@ zO>WCpxVJy@9SF5@U~miFIoTapaJu((kSsb->RAl(JJ5x*p3Z6v1zLzbpQT9SMAXUD zrmduEYgwQjSg(W{Mifl(uiNCYx9s<4;ySDFmU1Pr2p}GGD%Rg1n9Cdym_!}3=9xDUZ zR=f@T4}yFPK<#8@47k}Np7S{au@cs-GXSa+*q&`3ZEP<8E8$7Zv`>TK zx!UD(oGjDX83^BtYbsl|-_5X@qP=`vX*$`PBLFX7vI`3He(n8vOhT!4bt2&7Kbos* z;65+rY{+oIDk+RC3NBk_AKr$x*W8oEZa2qJZQaBubW??5CgSyPxLxZhLRoIalt6{QO%WCd1 zB0!Teq)Z|4;&|7qV&?d*&CYTKnx;iX4VavvQh|SU%BG{VZ}@^6YL~TJ1#JGQZ%S6E zWvw(Bia$N0ORh+YnA-6;I8@gBeg}Bt2wRpmp_hnI=vvII?1h%oB%*PbYyA+$Ki@WS z5q_s=Cw7mClAtruGvP%fZ$T`rJqkt1@_QOo!fVWtJWW>is?7|;r@qxK5=z5l-f(lFU=6QC7;k1sGFzf_zbIB zwOCem2f9_&vCjy(9)TxmGkJY7Ljk$)IHiy(2Jmtn3+3+(yW`0%2G8lzPdj9pN7-D~ zXByg8KzYQ$zJ{A8Dfv$K{WL28(_Cx4?}|g_rFAq>f}Osce6Zc zxDjTJPpMheuhGOUutVUmUj5QVb5-(Kmaq{KkNPvY&T9XQ>f1zveK zUkuP02Th+tBtm1gZkv0FrBsj$K4zpz_BiaGu!jTtn&zCply_6b>k=x1U=HJ@`qTx- zTpD$J&;3IMFVD|iH=0xm%R;2;pn(7ngBmhsIdylDG6pDwo^>avtM+o#Y%A|`z3iph zUo~~y0N$O%zvZbQY{Je`Wu7}_s&-)8hZ86a8KRrTYW=BS)xDp%Y}ow(>B*nVUK=n> z%{y+RT;te&#e}m`LIE4{jMLSUil1_3`>y(DHDAbD8Ln5avqP6}aR7e?@v*?)HaGB} zNh=Nxu?pw_XRn~WpSzp!j=Qzc*eH?f!go9CFX;Yfoo{O8w`n!r# z+(-D{*0swa^Y&Swm{|vpEZfGFwr{2dY?t*5E0CAfTh5i|D^e}oq!tz$X5qWpY=`XN zbdrg4w$TFPWZWntC7~(PyUj!yq_ApDbZYNxw4F~1aT&FgdMgV<_OiS26mH}ab1h!J zoSJyJy?o=f^Smo(;XL+omXxuxt%|veer$THXy6g&wBR*&t z+I^f#BxsfH%y~b(+|`%>RqqT+67I1(XyU^+xu>?WVz>3$opNDc6zgk<*oMHJ#)F55 zSeOB#GXUNX*JCTFw7ih7$M-5stYI#@2hMq#WydXlh>%1?^2(O&JZNs>+Skhk%R9>C zer0m^!F}ofjQqHfUjA(jU*oaxp=gW=-|931yWcFBR7C%1z`*QFYW4(bPnyz_d|{pt z3yPU4%B~Di`R&F7uOUqH{@8a3aG6Iv=5FZa7&M(|Y%uP-66x3<5xy@f@Eby_YMpAZ zKb~5hd%2Up*I|G>IBy5V_>^qf&fISF*NcShZ9zqF7*tC?IO@Q2N2SG5@?)aAtJF_= zcEmX421b<)vH#{;*JC2aZ}TI z+QHa1rI~S49ggT9adnOrxG&M!j20+QGGsmwfoNCyORUSvxGmJl15%h-ib~)A&>0fh zZzi&$s{1wxilHvmuImYn=(J-f7<}+X9s`Bd-bS6g zhtdyyZM|s)&VEbJW67D^e&dOLB`ZYmWIBskBySK<2wRgip1k)^Pk+@9O!XuC^YGUw ztPI8ozM5)>hxVVPlbdTB_SBdyqDZ_5fZLuy?;^I%)hJF#V(bp9a`&Km(-tE&?e9~hcF5Mk4 zIJvl%n(oX;1}fcWb1kt^IZ|`8?xeljXe<58Q~o|I(qMd6FAx{ESo)z@8I?=gSUevOEK+~_=cVTf#;V_9NuC_- zek$81^P}nR9UGYJelw(fuDNI}g0#0PNi~ZUTCM4?MJVkkn|megXpqjA`vNdWG4HPh z0<<@n3Q58b>FiT_a$;D_P%eenIw%$|FFVlLEDlWiprLJ=1xW4KsiiC=rn5;&J7L`SACUe?~JNy`OV=W-SVUJ9kx z`uY82(hi`;Kg-Fx{XQn^ciK|bm7EL!U_w2Z8zdP-`e>r}GV>0h24fFPD8vCDgnv-7 ztIADPFMCv`sK}XQL-Amm12XAF-)Y`4Eb5n3>-YA;Gq+CD8Do&}mhLURTl%-&t56WV zM5q28;yxEa_w~hk8;+OILV|`4q=oHiUxoN>x}HHp=4S|Vse73&On8u)(+*8)oLi|D zAo8{%vk+R{dVEpc+2chHp>*=@dt>AVh?ljW*Qx~?s6rNX&o<~<;g49{KJq2cP_^+% z{ti|lKii-O+d$18K4PiiXIrh`naz56X>V5I3DdS9rOfTro{AM7#d2=FWu9>Ba;+0`7<~c(T+rB*r)3n%rTk3<)1ufV zW`=$_kbg)e4z=fZPQCo0SMG9;F!huuYL{d1%D@_wtT%NC7_vAFn9a9Wqs*YFTq04n zE0!(6jU3T&*oD@KWrfJOFO+^6vq3JAZ%rp49>gQcYFUHkPbUf_s=EAc4d@82$vzczXMe~By z9-Ir7E=!)IZW zLxd9;KY8IeRxSW1^CCmEFW%C8>)gzvWD=k%(J9?OIqavJv#s}vE;G!2ZoD_w*tqs4i?bohL_w8|CtX%{~y z%6f(gxCvcR#8|paKYenl|5XWRqes%lit<&$#(;ACk80rc5;>P9=~Wwyg##0`qk`Df zt>61Ja_Z#{WpX2Xg{k?Q&NBNyaKbSZcsPIQ03FBWdS5MRA2L^!Gi$|(szDi4C{t#c zewrforsvE4litTn-@LY!O)}h`jl$ku@Q-@t+<#Rzks;=TtAvXcC|oj#_1;RJD;1bC zir6Qk>p7RtM(uM{%x?NQ2zFW1<`|95(Xr-qcWIDu6w9R`JGDo}Fm|nsZQn6%c@?~3 zu9gIp=4f1crt@iq@cA=KQydYcK=bGKZsR8;&h>si}}`a$n=N~otjSM;F5C8^uAeL`e4;3 zIe)xvB1`FLz-YtnDI zwB2nfEVSfwPeJ#Owf8Ve=fjDOl=8uiz*#|s&6?b#vPMg-VXyu&;(7a)R&zRQ7e$+pLR_qWWP0vU(d5)7cCsFKajWQT8A(*WU zelC7iB6Ffgx^J1Rokt$J4Vo$IJ<+5raDlPqg(Z!fGPgEw@_XafiG3k4$7Ox%!!Pfz zqaVh#^HoF!up)^X&X~ZorCG5$*sLn^@Wk<)T)jH{kULD)JKO11SoEr>qjrH}D(|fRTY?XYL;S@&&oMErB4oboo79}Gh5aDMzS9Ih|N4V|E(v_C|a;FiK2>N z4+4})$wXc4=NyW-THExUJ$>HwNp8Ju(uk*xlO{yAF(MZrU%U?UbOXmyysu$ps z(Wg-6m0lIF_x)bs+m~PZuyVMi9bczc-Q%u1z*1-cO!&_qLF2L*Y-Od^>3nhC<0b2k zMrl|sM1pX?EMa4fKuHp0qn>&z%JEs%KLo%Tvin@ zAO-vq-4w$D!$!n{6II8m^>*X!=G(1~Ury@}Mp&;@9neHsnpi|xn^;BJn%G3yn@W7w zFxW6uioh(Rp=7;-1TVj%98cpYU1Sb=h#Y>u>=H&cNyus#o7#<~Z~T-4g+(l$WI>LB zQ!Hr{i$Ms#cAbE(O;3%pC7pz>cURC?p2H(fe?)MXM_M^Oi30g2GFh&Io1yvVnb=xz;|tT!qi#=+I|ub@su?Rc)9B7p~pTYBJ6vQ z!bCU}5SU!dii>GkJf|{4_S$=mpum$9mxQp_ALDT9Ye`D+UcRKga-SkLmmm^HSFtKb zj^pf+(&$Ku_fr(oFK|iJKKC0Zln>q=kO@8Fa`MZ*t&`u%5alxZpiqnE(F%cFZNS&f zAXN?wHfhGhjcrMLBi+nMbVb6=Ot-p0a!VVnbYXK_J82N@IIb1j5X3gee@tbG#Bab|g6lh+ADvr80j@1C%V-Uoc~B`a8jkoHoRfL|TTrB6l_zgpp? z&Lqr^CdO;ZPLKz{oMKEQS+B3n#t;XoMxZF*CxX>~4sJa$K&L1~P^;-h4vM{HVqY)% zgvMfu0vp=FE&I5ij<=d5iMWWqirt7*HsOl12deUNVl?zn_z2n4@azZ3m?Tbre%r{A zEM<$i!=j!JwCWv`HXq4>_QNe@&_nt+RSu;!C#<_#e1(?{L!;Uijov=FZKO#n$4S;v zDc72`*Qd;dpyNpnVA}m6?&!>|~G zngvwg@`q3W3y~;f@zag6$1aKs|8%&}O6SAUlB?SdD6mx0j@^imFJ{#xW<;`891I zb^xzv;Hpd-NsKgb$I(8Id1WIlHjLK#9VeLoU;r;A0lFLPHsb#Bp^u{kv2D91ZGq`h z1EsBg4N~I77%P*kj}g9-_B%B27t22mO}OK5*4%h3ZLKUg-2rY%S;X>bCEh7(+UK0x z3Y>A69c;--ZTca=%tSM_en&^?3JhLh*e<7>l7v&BbnovyqQ3X?Temc4L7q8j@1R*D z2Nt$}P!S&BNbFRV49V>At3G*5n%-~os2Q|qGC9u9?Ih3jFvgLdsTRh492Vhuaf8dR{Q&h($tF}c0FP)FeY2oyw|=)?p58O#qJ`h zk@y!}qn2t6z;OkC8FAM#-@qVz>5t;=r*n(h>h_gp`YCANEEh1Ls%jQfrhU){5om5; z3@>of2WKGKPh&l>K2uiB`VFif*Rw073z1-BP%}g9L-fE-jBzL!`)g|Mm36wS(7u%m zb(inB{onwVlo}rT6&3cKCcsFrvMPtq&bZ+F^LUFG;yj2T@GZ)%P?cjQp2xca5{w8I z$6JnHa3caql=25n@5>vVTo#h{ZoWZ$k|40DV>?e$?O{vY3=LjMi@5jM2lxFiIe{4m z<&-`Bx6jpw6+}>R#M6zV)J=Nt9>4t>=eT8Y(cNxitT2 zS%SGwAV&~*m`9^6rOYx0IDh^n`XG+hohR3KJAdt#eJXE5|AWI&Rt%136;`n;djNx8Mx~K<)evdvNN{>Bh#$Le zrtRU=Rl}6T#ZCkIc;NvbroAx=JT_A%>4NMVeq{UZ6DHcu5&)ATL)xV&*X*jS{GIG{ zOkQb%BMkTD_kg^VfB-@3vxDZToaZb-u$hJZ=QR7J4He9}&6ta29S~PlC2zHZJ^LgV^OjV91baj&D}F^P9B`n80o zK=%R!Elk@F)S-z`sR#tV0WH> zNB`w)b@n}$pm7?iRc+u`U>{mGvg3ab7jF)~gdnoBJBFi-D3l|K+O4U**Dwn{BRflA zc6|nJW$Q|ScRC5vBziaRv!}7NSjzzXx~zFKls|_?TGxl}1Gba|1g=vhKfmPeH=h8p zrf1W;H^|P?1-=`Fj|2bBM#=m!3wWpc&=( z=A904gAZ;ROXa;>ptC9&H)m?vmfwjlnSiaem3@fX&BH`i%Gv<`^DTIOVD4_S3)4QJ48X7ZHy!_b$X;+g4xu&P zP~;4$|Cyne{`GUuvyq(*yE(H%7_EvImV8m?s7RngTqq&jSA+bu{-)!958H1J(I{3G zku%hNGbOPFIxsC`e-+75p^AUrtFjLiT=i;<0f6l+zpx6AHpJ>B3VZs7h&QdyM zs&)vp)hFY%w#IfHPip8)L~NFJZ7VXz>Bc8N5p5-CogZK@x!TP5Z1*R8OE_L*o3H-0 zN+R=bM$vgF_V~y6ynqQkB;zgTDQP!JIID239+l#0#TGoa@hRT`H5&}+7AA5 zBmym=LCJ2tl>W)8u!n=)cDzY7A}9lCef}{wv|mk9OQ{6er#Y$$_j#bF#{=-4B(1}Z z^%ltfF`nvHEV=D+61Dbp-^J+(*)h(sZUJ<7M|dPLE8;(m-gX9wah$Na-KbAQu(K{h ztmo`r=Z5CbZ;WMAG@ZURKf`)BH-qL=Bj=p^?$FVZZC4wYoCBM zI!qP;-HCSFv!pUlcG!YFJ(QwcM#DGIyTst^G?apD=nXf6RguHf8f+(>8D#hW)$VO) zNJQ&CQg`sV(_B_MFoby0T~Oyz*_X{=9M5ZRinUltFsA!lD;Y{&tH^GgVm zS41qNFXm@mZ`ZtV^@<7u-Qo%{*=KFiqssR3De#5i1%)c6BU$is&c1I>lVyZB>s2E5 z2|^0=|LKpXWDMHnA`%=lOOL-Lk=~+rb33WsFz$Zho>;HFhOvuoNiVu)(a_>kC%MD92 zyU%~$jcpepHMV|LSiv?Nf0WTj!FfsIBcBvWl--A4Z;-&UGz}_W{g=hM$V{h{m_>;EpeFQu+!Czk* zf&P)?bvx{Kd0^qMf!+E;CpD(Y#U;z;ZNX3Om=dC&6-#cxk;Y5Z=&*`pRJwFO&P^-q~n^B zcdtmzq=G6{ZiG(a3<$lmf{Ogo*pvgsV!8qUQV^XAq}ptnLA$S~elbtIN&IflIl^*|rX0%^HE`6u;_rY|a zzS%$nP=b(GmI1dyd#^Az-&}2Ccs|Y)>+L365$sR@rX~*u8M^EFKb*Ht&cL@M#n#=M zu8~}K`pz;t608f<48Cz2B)JBzb(b3+*fiRHkvGV5T<@Z=rPC`kPU8k^n&(bC#9WanBikD^yrx*Q?f9lpIILiFr{E8W{eEHTq)9Q7CzWAdrtz!k2D7t8{g_c9 zI^4it*Vrbsyb*<6kdax%s6^dw@oJSCG|C%%R;XqS+^yV7J5)XeVB3p7*tR(+GV`AX zc1Ul4pZp}uxMEba&JAcFPlD_Q=aL(!Pgy!DUiZmr@a_#Rhp0t4X=;z;DFA(PpSu~< zet;43ZV5pe%RC@^HAwXb5B8{;{nNmI2X(Eq|DCD-s^5Rt)c^Ml#Uh=Ye`G=gKysw% zc#{5}shit>4lp_<&PHL8K$!iekP`dC_}XOzC$kP}yDI{$!p)Uw8~{WhEp#6#^7X2S z@di*%yF4+xp!BI`h~pc)P~ap$22iMqzI)PV#4u##*bO0iJYMHGw!4aqQ>qT}!PD;w z6L-o}&1ZRsEy;r24!=5F2TI5rz}6Kd^*U!O8SDYo^E3A0WH5no%L*fYjl&JR^&9f# z-GWxRhgABOqsuT5?C{jX3oD!)Q5`#(s1o7j&UreXG>LnfKx~`$Dxo_HZu-6Me%0M7 z!fm$g0Ri1Vd6a&uy7CLuLZpKa?kR*F^6JIrMItRBm_7)<~t(P=<~D zrCi>9wVw?56#;|;K87>fs`4vZdhgj06C@GxM_K}trsG@@e5=VZJ?v1qPPXxFvJio_ zX6^8=h#&<*O|vG$qf@%akeMRg39r3j`o{cAZRSAYSx`|hZd#%apWF@%$>Jn`yzr8S!GLlNCSO7es28Su$R2A zX%t};5${Kd?kxDB^qbX=&#Ls;tnsp8lr2XM!r^TpWmw-e0fQ89cSVGgkba(xd~e0c zbf*QriF}lXWtd@TL3Ps)n8o;hnIF@%(3ed4c{u=po=mgOGV_&HZm+U|QcmhGE1p-T1mAp(a5 zV$J1ksv1wV{M?g^wI{rQP|=rD>x=1faFh((*1QtgQQ@$@6C@Xu`(U+Xc|vBI!}V?) zZGMFd-PCq#egr`dWQ8JGF2-HU?$Q9_yI9WWlK!TGsdmew(<87qQtPDER{u<qv6ejAOPyzx4A3JhNcfA1Wx4H#%Qh;43J`4^^X`#q%iR8$>6Ihl(8C zFk08SbC-w5r*ubz=G^l<_>5k2yTR)u{#Ok-%mepw2SM+L_V7+){({!gF%qo~C3u{P z&Le&c6cAdq-+ssgcJVUxpYwH_Fsi|c?$0+n&A&*D9F9k@Q9qi$>oJWOH&^?W<)`1*w#jiN038#pDk9esxxiPGM5ZHcmUGXgckVuduNzj2c4OVy4oSxs z4me)(2=0IAA1;?a5-{-_V3l9&#;732El;*vrS9Z3$W#O|dJ{_naO$s2dCODM*G*!W z1zY!zfw&`@v3URGytNyinGkH_BX+pb4ll1odI=TBcFaIC<#1u?%=3F6gL$Jb>S(|E z@n|G1nz&7&TOt27^-g)>)*fqujsmGSfq&CF?K0E9x-VrFXHf#tXo2V+op;{*m?f9s zj9i@i$OvG|QL29mAQVV0jbolU-T@CU3T&RlMqoRu)AFrMdHPCGGydxFXuJeBtxh&F3L53 zlZIP!Dz?mL?ovP>EBV^*gERd+FM3cPeL_cJ1tUNmz+NbG=BOUl;R9q8`_00B*SFKy zy^zC0@Kf=vxn~bnwXdzEz}L-y6mG9f=0}aoa0^h->zcCx*5}^(U@Wm8ib1Jh+lQ!A z$3M3TKnk(E@ax3~#NNd5+gv%qv}^v@3_y5$i|?I9VSc2p(LY`+pxmG?OEdx?Z6we%nO zMYaL&$5MVLBQC0}NVi9T=@J6_IV9iM#CE!}xO}GLQ7fH0;D4Eba58PiyS7 z^~+ZiRvlILRnER|>@np#Qaav~6BlL-AW#kY--&Qk!zsV5C1F^0_{GVZ`O$r$U_qOS zMgE-Yt|5~kz;Zr1R(ZBkvsZoBs%tt(ox*W(W@M4=&R?w_rRmW7#QphxfsD#`;mgdG z>BGV7e;=4;nW;l6&(4C$YxH2Yk>JVpc&kw$TIRTBY>|r0r}E6rmjurlppbn{(q>*y z@dv#k-ag9;bP?BBM1ngY({lsvawS;s>@QrT)_^a1j+abym?Fb&4liWJ4A9jx_dG&4 zwvHFTQB_$upU}0`*`+J~2N?M<=-ynzUUiq`F2(pg;{5+W6z>96sRlQ@9Xo19L#y4a zo#l8oM(Kx08ju;DYuA-GrjA37HzqglwwO=6mrt6mh!-kteh#F+kqM;hRjF}JMOfKnSHzVvBmfQCwlmE%|Gi{6YW0$>Yoe# z0ixE|l*GRXc??NM{KpFVbIm{U+f30B0(yeX5?-U(zl|U>d)IC+Kkx=UBeQXU2WEos zXZuTKaQ)3-Ya{;)$X!z^|ANHS&3sUdb#xz&X{IQpVqAf0VsFxDiATMIAVTu?2w*%fQ%k< zGdAC30KOGyW>K4OOw<4p>e%o@6MGpbi}Hm0|F_)Y|IDfpBh#-xq5}yMhp{+%Vt$si z^LHL6dCmM_NDBpD;!t0C-sf|3fxeezj$o`9`mi(6`6CrJjmcb?R=wx0P+ijTp}=)T z_DJOXWT{m5vQ$=69F4-$uH$N^5KJVj3qCfON5~%wEr%qMlm4zlDhKWyC1d)P@hG+hKKNMbW&AZA2zO;7US{fLzMpl|crdm2p&uD`c z4cdcbqIA4|EbYfPtPT;33L==k{j8dmY)P&FJ`iZ5@Czc(3{V?v7yt?Fd3x4RZO)jd z4(W@ssBLo&ClWqs|G`9)RNQg3u(^Yk3_gK^F_WEVj`e^i4FOLA9LwVKY<6*wAcC>< zF|2NHhcMA*tgZ94`%bc)$OYDX=2h!va^W+xF;|B zJ*c+#wfnE{ovy0W-^sn2Njl$l=Mx`UUnWv$Ptla1>~7@_4`?0^noXajy6|S-}j*N zvFHY;GhD|xhhcdt(!1OBfY59je9*3Iu#}bAKsxRh^3(O%>96_d$;D8EX2$a@K!Q*F z;lm%0*ab~hG&1!y%I-M$cOh$!#Uz0Wkk55nEM3X~KBe<%crsciy7qXRYMRX-%_tMdE1B8hwQuV8b6ot12S5N!e!U z;)YaRCP6gIujY8J;K-OytC< z(hEm$BV%dFp2b|iaQ6$<)ib`$A@i0)ho;dA-_7y6jlfs&qBf1ZR`XzVCg9`WZ+Hec z>>;%Ms$*h7Xk*CAvH~pRPO)RQ&w4c%W&KyoL24;hi#pA99h6@xod;Wb99{OxoV_V| zfuUz;c_HeU)=yq*HE8)klAOGtzss@aKV28H5xnC;*^RLFAPt!`f}_sVyC=DEjYw%( zACQ9gHo>-#6e-}aRul7X55|0Nk;pG|bb5#`)i~Y4vwX8=9X-9h{ z+K$c2kR@GT@>06q09$HPBJ8b;CdZhdv0u85_*UZR)h+W6n^`y1AtBf}X_Je6b?5Dd zJ->$ON_+UtUDPrm{_}kq6lV8Xai}xm4f?8D#H7isWZJA73`l6;gcUUqKpTNL&GxCv z5uoffx&RMX^k&!6=iQhzq4xWAn;zeIYy4Y*F63erF+33A5dCa|Wr&A$d09N_`8vzx zGj(I1$OyBnc;@c}UJT&h z^JHDmMC@>k(tf-G5_ciQ03?AwQ=PG4e`6m0BwAf~fG9bW+WUMIFlHXee8{a;>i!Ot*Li(#`PzYSyS}JId8uCOKmW4|d-ItMP`oMQjG$_dSv093r3dye~v&p=lG=)?632 z`Q0t@1=I_|-daVcj?CY+ynAEh4~ik#9FWzQr?piB~=^wj6=^-+ZpK9~AD zc+z+}uW zcyIY6wMVp5{)t5EG23-?0hm#b)~_bjcBy`!qD+qPAGfA}z?FLs1O*XeL@ln`?&Gx7 z(lhX)Mc)d%37g^GzSwl0U_r$i(8)ny!M$X-`%3V}zxjva!{M-_g58J!0rM{d>e# zyuYeV+&Q|32H@Lwkk(M+PMi)lVMp33ZRXde4@Y;&%MS0?JON5}?JIK@D8Trp6N~7K z9qHPj9vdcM=({Gk=SJDvRT?Sm3ms(UfrQkIz*HQELxA7J%e4cX~AgkG>#Ams_lb_L#aDb!$J-w&$|W&-~`v$AWY?x+z!d|Q-sVI7!!ww+5R z)GikDu>{biYs3JIiIlB6pgktWcYn3o-%(RVPKu^CA4iBx*!tHgH?>Q-SUO2kzwP!d zJyMI4o`DSu{-k_L*FDc;MipOQufm(uNF;U5Jx;Mu^_66VW%2gwdEakBNnJ^E-~HaK z6d6jeVCARmNxopg1uDN1mSp;EYCy!r&1|qwl=3R+D=0oyE!>4AtyMQgv$?hJQh?1) z*?Sg??G*>@QiJSrq`s z8eZ%%@y&lLl?WbXVnU2`6Rv)+)%?{0ES-IoPqVItsJ>ZlOZh@g%DAMaTU`FxQY4aH zwrG^D?>hvJE{!F`;L<3p=8WH>KKgnF1-T*o8thKLKj|C&aNGzK^&$ro+fE==GmvdP z%t{mBDc4jqK1GQ}PIKpedWj4#TS6)Uyem9~!$ShQ_@Uv|=D z83`B`SVyAf8?cN@PYFf`_}qXB3gzt;5W+?R)FimE#cJ3Z9L_0tGP&>nz8Kq$o;M_= zDJiY%r$f6}+-gkX{`-D_hq)r(?0&qXSVjvMp)+l-Vo`FK{{ zQQ8T(&$jlhZ!JQFT-Q0hR|@OnX>{8-ai+8Z`5ErIMZY}>>HoUFexn@iGA$bN`sh1w;E_+3(*mGx8+Pm{JkdOl&oax8)x<6%} zwuWW4Fz0N`kx*ih9kP!Vaa=9ss}>;zsMO#VgfwwVSquUW? z(8>y>`~6Y^3$3Qu)DZ4qfh~z1kMcwJMPUny>$lB*uVwT2 zd7p_H@IC?#y@DR>W8uN@5I?LT%ZkleCTKVAY5CL)7Z4zk0(j2$&dye|k}WeC=DW#7 zl7O~vcIS(C`&mD9by5-$c>zsu1O?9TanIEIQIds}=4lfWxEgy9ZX+Rn*e$(FI077X zQTv*~YBet7Q+wuo_2Lz0;O)IwJk$&!M7X&|NIHE;QVrlz?L9Hge@q{x zEle|61v>GCL@O4$bw@yhHb%;@-@B;`Bd2m+N$$btOSeF^I6DCa$ne~sei1ia-z$yt!LpmJ6k zVp2f8!6RTt$NX{<&?|zlpTNg(DNEMh3w>B^jHceVv_%Tm8xSnAD+?5U3Gpi36R0f- z#N~MY?K&cE?_Jd9R$c@?g`#Jmm%Eh7c(~|mYL%4&UPFvk%zYw?g2weSlE=@%gZ@k? zFu+4|dWZ4`ith=QpU+qF`Jfx*eibIxakfU*oNqxE4MaTP zuIMWHa?y-z-}+D_XMu%$0mQ5fi6m_g-|;Rg1!f5=PsBQoZ3yeQgvTKSh^Z*c+3tC} zeI$@uJgdWa?eb!&5)g8-48*Z7;lKr}P;!BPRSnr4`-)1Ysd+$C0i9Y)qSv(*#ZVHj zW8NfHwE5kG-o_HrHwW`VdOGkjS;3W0x85{#zk zMxE1uO8E{eBeS>a!3d ze*`e^0GZW{gv3!20^*YaK1QaK#su?QPAWD}qz(L;>PlP@*}#tE16Dh7Q34#QUuwpy zdHX^!$#L?KG=+oT;UT9viS;IUz+RzRSztd)TMNFIvxee#G}e&rzB?;-`v&$Z?>*sx zIbq-Zjv^+bju+=TmJoWru(#B`y46B%1Sw+nt%wi_#Cx|_4eDE{UCDlI zH7o(`L?NjVCNkwWCR#R!`SYau1*CyllfV4UgQ-WzA%bAa$Ln8dJIY|R@gb)!s?&C` zUB}m7-Vnf>#sBPk>cT~|mL9%MT031P9+U3qBcL}8ImRV5*%qP;kQ80n1xAIQj#qit zpwQ-=aw%CJf=;aCD8^67pGI92pgQyj*eiv@3mA`usd;|4-@W}a^*>!$SRj+{wy3=3 z>ueN)57leh1}b6f7he`DaqR2DbS*qGnR~mFC9PISBSP@B-qm`s-upj$|J47Pgx>${ z-E7K=dwl}443M)G7YH+a>=y(p0N5J`>~1Se`+c%{h1>PT&h==9W#&@!&qlNw_STx` zRd-v?o;94-K}DVdyXTPf%{WWlr37MX`C=baxA-rQ_a{ zg8t8(j^@>0E`sT@v|WtY#|05>ub54%*E1E$C!DgDkTlVd+&1Zc3P3&A1w>C-mn@fe z{u-7af6I*MYtXaXS>~bBy;^V3ipx<;*d$@zSw zAQi^KzgQbTjifArD*tltnnKIUQ|?Y#s*4E|TB0UaD8=TEDov>XW-LTnpYu&IbE_Lv5{>$a^krIZ- zCOUTVc5tP_C#tYZy^;&7GhaU+fX$HKxw?#J-X38*FS@1*iDoWZyF2AL`oodLV&~2o zMclLv9KVwyRFI1FIlxuq>|SusxScKKiJbTTNYrSvF2CN;Shw-bKmntEnAd5Vgow^r z@b^o*3!zXyzn0?CVA=K{l86KVx--vQKL@!1GD4|B_`sr47~^AA;mqNQ?EKWs>VQGB zrzOC)Y`8vd@xWwEuqYXIvN~IOc(XTBk!2xH&Ee~hFAbYkzg+FEDI&c~ZEyrr%h4&3 z7H^P$v00Y!Ud`Wvo6*1p)SZGeN^-Z)+n0X2E-bej@r{#df4M;~l=604uQh2ffNU3_ zC_g8;Y#UTuNr6v74bNtdFkpK?%xQO)rLyOqKS@Khs?Ab;R3PUHA6^k^ zIP6{XKcTe;pP(XT_Y{840~gLLjpUjaknL!@itaoahY@t``Q!IHoK+8=VBiCF4z9eR zZg`?Bu8E%Oj#gA@tKe(4S<|fS6RD36{g@4~aSyHFGim;pZDks~{|}WZgdrNch_L2dn%{I6 z<&Ufr?EA@lcc9z;{I)+v$FL5QW{#K0l6}wajW15Z>V#e4P!TNnqQTo*>EKd+@E_ zj7V$#$VdW?BW6pa?$J4KnO9gU$_~GUS!IADxZnECU{DWHVI*bAmYwXzHg+jwnF=Ce-4xvN6Vaw8g30Cstb!dc;AYk1?ra4+&ZGHo$mzU)p{b9 zS(DKCD!nr=^(#7yYIjyIm$1pVrL&kcRFF^4} zM${SGrOOWJ$C^*l)BX2}(m{}^K;zQG?O39I7xhE#N^DFKyMEA8umo?=rtl;LEPaBK@UK82g&owhoR14H~+6tY9^A@BKv{a71@Ubue)vdag z29J8sCS3;PQ(a2Km3+E4$f1H1N@bcxYNEuPsP8D6JnL+f=>wI-tgc2xneYu}?xwRrZV2j=LQpI0RE z`NhYE-VFi$F?YTnAP9uLyTL2z79Ar=l-`aBrtQsf)yh{R9q9U_G4Hm27N~{V=KRSyz1db060@qi# zjEg^h;&HCu=!UQASu8;O$sHQ+H=mNtsmwal|H*?D8uZwDT9L7WJ*tH9GLAS(#R~H3 zlqi=SEgq1KJr{*{2A{HJdMLnqB4<7!zPX}W&KFO9{+y(&TR%nNE`V7KI`^Fg0vQKr zzl%x?IQ6|GO9&a3iJ~*F7=1VOiT|Jy7l$4#H88eLzf4Hi(_S-T3(L6RArNQ4nktf6 zD<)_Roq9s&&9<4UaAlf+mD?oOfmnsAhg#p6Rq1k5o*UozCIzE< zg6-;6ci%_aWR1XNgBsvXB-N^ukIF`TlI#BY@QTf7L<}4Atl10bS+Z_%HhhuuPfk+0 z`ArqHFoT^DA7>4CvrXc0+jLbw#2poDFq0@7z)@AJk!`55XAJX|2N`;7iOJtp@df&> z*3-{TDk+iYDe!}i)aEZ9qK1I%U5wY-_TxM^Z`6;Nwm42e9*k%Up9M%U z-;IOMi7<;+r#=C|mS?@e{m?3l)u4B0jT{mt6J+#`%B*J$qtdm@&v!qwx42TP^Xt_u zhq4FG?Y95ySyqysjhpZWPLmG1&+Z9hp(OWJzLvQo8A(>*KERm2$H(yR&53 z#aZSI54wf~1^TB0W59tkz?Cu^@j%ZJDT&P58g*gl`segLY}az>Ao8Deg^9z8TAaf$>BIJBf}KpsYAPme04F}X=oPuAVD_<4|L{=m>!dT0If|3_$Bk9F!ovhO<+?N#_#lCsh(8K&JjY8!n1H8?ik^vUWlX1a*%ZH&t1r6K>8jNn z)ZCYvg>Y}$HhLf1Gj_buaNIv+t84YzqS=5LNtUEl@mDCSg#~ZTz|VBL@I;t;hZAr~QqwcTucrEQ7SOago3>r(A2QGo~3 z&2F_}tGwp|Oke-ULS@nNmXDt=`8+;YimqMn=%4Ox=FdEF`IbGQ^R{uZ^6^HfUpHT& zGobid6YZLY;b3}bknSu%i|fpm{I8WH&azFN&(whMssF09;930Vbvg?x_?RY^(r&3Y zvmP2(Gs~;gUy0G(e3kd0=sQCSYHs}qe!7E$h#FYyi0A6Rxchjx z0wm{GNe<)ynNq6!Mc?&Bn?rU*KDJ4>dNiIEdE)ud=#0dOXW`N!uiqXk{;*#1YIh}g zig;+)qkYQ}(fx0_K6ihql*_=>XGGKNOAZZw4qJ0VMkIAIyh78x?7~58&pTyojk$Vt z0#^*two)b#%@CR(lwZ~fU0-S4TN^1ZQjC$v2SjX-nQW!WA9;#O1q9ELDU?S0v$=00 zh2%PJtc5&*TLH}2`8fYCdZN)!i~V5bpo+REs%`BdYp5grW%j5y|F_6GU3I8uSO6kz zu-V}fulI6v-%QH`&RPRuS3Xsb&CwmC=i%0xDW2+3_HGHaV##sKr-MC)l%T3F<0ou& zls8+aq=?5#{panxqMt7WaRL}S5(BmM*q{(l65(JAgExG46oI>WVAaq{1;i>%^FUQ+ zG~?wS7S=prmra-#ZW{8`UVOhCh?Usk|J3C-F8Di6iRa)gHw+S+)UVDy^T@Q-G9@Mz=3o?3xX=HjDnH>~8Aw!O)cz{`~ty3?~f5BHROr zb#gtjI$gB2+Vo|VAtyA++|0Z#s@iepMsd#({aTGE)UBh(oi$(Tni})<8HI+x+xkqp zc*g>Uv*Dx6}M+U3r{>r-s^jKfM#(RCJRf$z0KH>$p&8bHBLnrTwt5?Vks5pCI zeC!P+w{m2^weQiKZww%W;(z)em#1Dc`0B&n#wH*Cpy@}S+TvR$%I%#=yfNRHag0TA zmI17xCY{`Aa*me}<}|qzAOK=}Kj38Q#A3>t)pIuk zQ_xE&4pdjYmSnT5)!PW&ZHV1qVzY7XtY=QiQZ&!?ND$w*TTu6LSO+#;@PQLc(}8Q? zd=G9XpQj%F3m-K;zo))y@JlHv zIx2&Zx)LDX)k5_Bp>>+;eG`&bbg5Rzwc{us>(h8;r}fwtPVZ8yW4GcjxgCjgMtgFA zB92P&H#ZA>V*w?lDecDgH;U>Bfz4!`*`zhpmSR=lwlsigRt3 zbsmir`*Hq_`4N6yjSso9 zj}sj?>6SU;Ektvwfc^tfx+E1%?PWn^3}8Gi8uA=6WI$~gkafCWP|NV>5$A82s{0>U z)E%>m@wpm*RHX7A#9o-k-RW(Y0{GKI_U0Fvcev;J{J>xM?|-7&(Ro?FC8?x6P`8LI zZog=g^xtY$00sP8H3|sqH-9Tw0f}-#T=bVD)gjG%SE0$osRH2YM1JW}{jWQ`_@#{n zxR7Ag$=koq2AZCd@5TQimw#Uwpk998?2y^}HIP4C%I~xPm-N=A{B2-p$oN4=(v1Ns9sHu!d!wKo*Fg96T~S<&RNcK#=@#6dtw=uqCoLKeJ3;es zJ@ps2DVhx)jtlv2!F0FwWQzl=3;F(^{MK9J3!f>h`N#!QmfK#PB$>W(-T2vaaK6p8 zXCYN}&k%;A69RrL=OzA0aPP)z=z6{dI$vkwZ>jjr87JF$E|lU_*a^f)i(M-}ynMsK zBFBBMX+Sv<2-w%Pi0j|qEDF=GFn!95KVP}1YlHoWV%)ARmq1-mLsILw>b^Y;lb4&ZNc(vcu+*WyrWT#iQvZ7&D z7%Q+@=hxOJJvQ?rfCd++Z_Wk!TvZO{@T$TfNWBeZ#RXJlO~3R<`qAN28jF|U1BcUw zj^`h63Z^(^%!Hw4|B?@>X^8*o`*q&w3_}C5c}zNFz2ljE#GaeEPdMICvo09>VUi}w zu%d*m$!GD*9P@Uo1P86pNWA^B^xDP8oxbo<*JAs!Y2wLH`eQ143j?gkFr&3m1dz3? z_U%fUVHQ02zASiaos!^&Vc3Fn_o=1(N9TI|A=qyKUDeF0TOaVW4Yz&SqfG-b7J;kf zcvS@(rimT`U2iKNct7<#Tq>y_Xu$BF3~ffbF&xE^M|bOHy2<95f$Qsw(%_w0&_P0N z2tmKy#BlU#U>ByRUKS1~n*gg z?Ou{wP(rQuPbuKs@wC9*F2454kbUy3@AjUBS_TavZ$??d21GPWKgSmZuj#haeN|u8 zqS|FGQ^t}ByW>QB!LM>-Gs^2^s3Xx*t)ZB=+Et9R%AywpPVs2$VRdLv zYFFvgBm&PkIx@3<|CyHNu-@@um#v*|;RFyVkPPsI#utv5xDIS}Iy{Cn>@2U=LsWL< zSQlq$fm>}#DNL`_^-RpTFQ^TSRfAgD|a>_8!pou>)4Dc0j8oNK}vFk!WxamQySl=3j ze<-`daq8Jgqbw=47}m4u*CN__MXelr&m{_rN9(hEhk~<^1Z-LPC4;7iviG&hj&2)% zX;r;HFxPX9t(LxoTp=&jY?=oY)Pe9Ec(#^`#?9~3 z&f(jC^Nv)O96a3X4+)S#3hYo+lW8qJYLWD}MY~$1UBR5&vyJkmiFyOB;D^zB&hzfU zQyoc|hV`EzQklqM;lip!WJYkON45=7{L{v9Gyi=#Y9CirzB`>$cYza>=JSP35t#K%Cuv)Ho2~8oodPWIm&l+{(q+og9y>K%v->-}y4hNxZr-1Pdo zF0aPmJrF#1>FV5o`QG>bWPRD~5TCpJ*JSpVJKVMh3!E}30~D3lJKc=>SSv(=8+%|5 zQ89jiF@AI8$)h{ zuNz54Qkk}r$gN{s&SnR(_VPsmEuV5-$2Q`9AtAHQ)Sm?E`vM7sVs))>^0At&-&DLcOrF!HHQp1H74SZU!V zcp`;|+QLURS827sMgNE1?9*hgffp2k!^E2HO;r}>WzdPy&X~&jT}-lmo~rkVZ}OR( z-4NwtZ_Xg$nH691Lv|p`V@$q5`+S|F!6Cy9t12BTcK#5KQeg_YSnx)@!YUiihQs?k zyL%e^Q<3UHleh1y<=GAl+%6zBB|dB@81}^!(mX9`c<8Q{^r$mZ90%s)X)%O0)!)nN z&|~|d(#vD(GA1*ntY>9WWi~TZ_d%U>=R_?5VA# z((L@MHCt>F%n=;uMz(}*L?#f7fUGlGzE^9pmrk@soxG|qr@#^)f1jT}vAr)A_Tsne zy<_U4cV+&?>^##!Mv$&#Ae&2ri*yW;n$5SG&jC6N(Jgo}6u4D$R zc+SMK+oo|0J27v?;~GBfy1X0~jKK`q`MHjEPV5u*!zls_VnwuinwsrvZk-jKu-P4Z z#r3`wC~uc_W$hBD1lck70)fBk>zk#Ia}3DUh7vXe_|~Ro*>c)I%xmSc>t#18P2DFw zg&BqHaY2OA+o|e&t=4s7yF+T3=n+y3fo1+6&uS;B=DnU=z3BKE%}TUE`6hnSfQTU*v^J% zYEeNi&qUa()TH@6a|zv>o#d+bt1c*E6*-}>8HF2nVou2n(9cp>Gm_RZUUKVv*mx$R z%6n^+YB(5_|KISY&tJ6}jW`z<802fNsCjju8rgQ8Q7A@RFmM&pa5uxk*GMdfpRxsb z2LC|sx4h)HDH+{}$LSqRzQGmnNScXmV(h2uYseLbS2~hCbq^;%!-%u^hpq=V`>*O9!*8~@K6Ehtj@3rdLWoUHbP3ULc zwc#tNQx%2{g~lLe^R8i1&01-X2Z%JPv@8vi&M@RzL! zrz_FX?CwQFf+e~Is)qIV!+oxxxiKorh~ro8cE*)2?D2>cP1{_N{mj&Li+6KvqkJ(9 zZm()uL^nFja+e&z2E4iOBVFIVb(8AK`0re%yy274aN-W@vN&&TZ*wV@L+)*KCcNO`1kZ#SD+}9o0n5NXYbdCGN6v}I- zqg*E4=_Khaa3lK%h1v2)x~t8E@YZuqYy0FAW6)aKPou5XBQi=%z%j+ zF5s&Biqgq38T0JR>+tlt(pEb|U$=jCI%#yc9sE#nZf%&NMA(eRIfuNO=lxWD z(hQx4^={kPgga(zl%!pY?t!}r#s(Kl+nE%{B_CMCW~qM(&S*#m0_4$ba9dHkC`*67 zabfHzBAxFQaRvu=^K(*odnQ}v-U@?0u?~TTK0U}G)fLy~)_Ga|iQ}JvDH!(kw%0Ab z&`Mow9+un6r&Kf!*bS=LKP3XsjmlG08#%yOKke-%j$c_WGBZ0^=4z;yG#u|2irIXf zsM#MJyC>yBLg1Rnnui&+S(fLGY)W*A-#bsQsqOv(!jJo?7`=+wjI~vO zVk)>xE8l5&hm!Z6g4`MgpNS;xsnSi3i!E#jbRGtciv`WMrw3||r}VM23k*48jF&_e zTkEh69L%RjZ$=-N_w^Fxoy=iMt8aiRs(XdbbF3Yh65d!yh-+{ugc83R=g{4|IeVrX zHg4(ybJw1norrdHghx*8OaB05ikG)-+S4xJ`740HxA0E64aa~SQ(a~h(Oh6Z8loR)R|Y}hqo$~{e;-F& zBjToZu4aUMXJ0+WC9TwGN^ig7WlC|}^d7D!ZY`&&;9TnTHRs)TQ9?0p{p8 z=ZDsgiy}r^ki%S_w@!vg^c^n&7vzUz9>%uA==mtuDKuMoYQ}iaj|ZL-uRy6db)S$G zml!d+cy1-`S`uPoiVy&vaNwf-V9c4TqM9pA8k##j>Ll+EcN)A>FaR>P`>h9&Nv)r7 zj5)AkTbmRn{0`ztAjoBuBS)IuZhGP+pF(fjEZ;4U-uCG}#&{y-x-gpz8goz0M|8U} z)k?j1g{4q?{x!Qnm0T<#EZzd1q0~qWGSm-Gk@h9&*0}_TAc>Q+vgmt2(8+YHj!#!? z90(G0_O100>F~^*DGN+Lfx83+Lm+A*HXoSaQprWS)@AYOc3z>c$FWX>xbrmg#7`eK zy8B>$wgWj!?+;cNC`Lt0rn<)@GKmA#lm4B%^|Ktqv&BDu+87!ZF=_W1OUl1|Q&9l> zblO)Ooe1&vs52Sm+^DzM?=BhbASvJPy6}wMU6RXJHO|UM%_~%)ujxwN;&+s)$3HP= z0yoNxD)jaTx4=`u%ysOS$u;%!0!5fPo!53s)7Z5?fwLS4KaL$lySV;_zVlHS1qDNm zy?#-!jP^p$qJvVS?M3&I%o#pv(sJIL56!_}GUR&9-dl5gtv4S0A5f0~K)sg2!X9_a zExF?-DDBkRZjUZK!J5m~5m=eUfK{Z0IQomgX3~cLRK0M(JVWV00a+WXlA5Sy``O3a z@y8c&N_Uvn55@ee2qnPF3Ju>v3vwT?eIOHCZ_i_M)7VH4L&MMTh7{5zv*Uis-t!o| zs@Q0%is^~H)_Mh=d>SEu4nLkFQz~OLd_s!de~!Puiu&^=G4$Qnm3nf!aZ^DgbsJ)WVz;-W#=E%w!QeiN49HWhbCODsoKz~el7Ysc!pG0aw)SoIoPcMC zUmi3GA)e+zD&HMroc*f9&cB#$7rUUzoRWT(9Jc&6xVtJ?UZVxZn4p^?S(IQB`&=@& z7J^X!Uha)L&ZxBPR8MP{KzECkbX(@-wHFowZgVfK>@&Hdt2=_*<0EK)ev87>N@^>g z@H$5oT`1tHFUf}}HhKuv?py>dGYYx)G0u`-H7*tF43Ya{YXB{-RzKXx< zp=nIyJ@gAiv8}#4qetNMMKHr$y9|4L#ER%^oyp65coi3tKb&f8cPr z33EyVDqaf)<-<{0Ngdtdi?-7mfnKJa`~HuplsVTjk2TIGrjK6bPL@K@8!vI4;E7L2 zlZv{QE@kPytg%8oac586o^EuQ!hYw zX3#6Ke97kcCt}LBq~bXskH;0=wt62$enC|HnIe3a8OwjMUymq_;jb-LQ7n)=h0FB4)V* zO+;?;hWnUJQ22R`c;k4)>ESO3#nYx|c(d!FRsiy)?9GXE*kk25m_K`nJVK9o>5B^H#w4gNrE6 zqkfN6floBtQ#S@>`T{}aeq+1D2*ZBHd{jx$s{9?oF~t$^N^NI_mdEx{vJhhTdz36* z1#jZ0iw$r@Nfz49ZukXd@ZsSi{eGY_Bs}truh)C(iL(E3lSKyoPRCC%dVv~aDO1=A zFIrEneSqqEA?;N~sh8wg5&P-8{Dolj;$xCmC@Q%6p2NtvL#C1HLF;~O;!EgBBSop0 z#HW(_BeS3OBt#zlU^_v}26A2fLlp|T&FjW zHBGmoiHd8EHP37v`e8ZZ7G`Bb($jei#;<&$!$M(lBcGVMf-3{NCtuq`bOVv9?BgI2 zF_hFkVn|me^m$08wH^@7lHYq9Ii?MXAsb)w6^Q1X{Bh^U>3F@QD>_>`Zjfx4mrqqo z$X=4Hp7!;eGc8y6qrR85+8#c18Ct9 zcfZDrd>iw^cEKrnuIkFAL_#=g%8?W|pX<39>C#Gfqtj&?+{PGlxTR(%GQFXcjsTz! z0ABa9Cmr7DgjlqGJu7S`xIIL1I_GB*7_ZF~8WODt`qw$d zMh@^J_6x=leYP4~a}$HO%~wQMdg%|bINZ$pI~k8UWIfW{jq8i%52z0OxIwCXqm#$O z&J)#Dh73KE4Wrkjb;HK~vk#A!d$#O^QTnEvd0Hl62$3CRDcGcP$#t-`JtbhAswGis zGQ?cm{ljrWodw+|@KeM~Oc7EW9Ax!^UD5?=v1T|@>DbUDceh9SA3s zL@4?X61O@qsHk38nezUkKY6DNW<@`ZXgZH(SvET{KfIpCDu Date: Wed, 3 Dec 2025 18:45:23 +0000 Subject: [PATCH 3/7] added Dockerfile for the batch pod. added miniredis for testing. added Makefile support for batch exec. Signed-off-by: Shimi Bandiel --- Dockerfile.batch | 39 ++++++++++++++++++++++++++++++ Makefile | 21 +++++++++++++--- cmd/batch/runner/runner.go | 4 +-- go.mod | 2 ++ pkg/batch/GUIDE.md | 7 ++++++ test/integration/redisimpl_test.go | 10 ++++---- 6 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 Dockerfile.batch create mode 100644 pkg/batch/GUIDE.md diff --git a/Dockerfile.batch b/Dockerfile.batch new file mode 100644 index 000000000..be6993b65 --- /dev/null +++ b/Dockerfile.batch @@ -0,0 +1,39 @@ +# Build Stage: using Go 1.24 image +FROM quay.io/projectquay/golang:1.24 AS builder +ARG TARGETOS +ARG TARGETARCH +ARG COMMIT_SHA=unknown +ARG BUILD_REF + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY cmd/batch/ cmd/batch/ +COPY pkg/batch/ pkg/batch/ +COPY pkg/metrics/ pkg/metrics/ +COPY pkg/common pkg/common + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make image-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +ENV CGO_ENABLED=0 +ENV GOOS=${TARGETOS:-linux} +ENV GOARCH=${TARGETARCH} +RUN go build -a -o bin/batch \ + -ldflags="-X github.com/llm-d/llm-d-inference-scheduler/pkg/batch/version.CommitSHA=${COMMIT_SHA} -X github.com/llm-d/llm-d-inference-scheduler/pkg/batch/version.BuildRef=${BUILD_REF}" \ + cmd/batch/main.go + +FROM registry.access.redhat.com/ubi9/ubi-micro:latest +WORKDIR / +COPY --from=builder /workspace/bin/batch /app/batch +USER 65532:65532 + +ENTRYPOINT ["/app/batch"] diff --git a/Makefile b/Makefile index 4d5263739..120a40dad 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,11 @@ SIDECAR_TAG ?= dev export SIDECAR_TAG SIDECAR_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(SIDECAR_IMAGE_NAME) export SIDECAR_IMAGE ?= $(SIDECAR_IMAGE_TAG_BASE):$(SIDECAR_TAG) +BATCH_TAG ?= dev +export BATCH_TAG +BATCH_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/batch +export BATCH_IMAGE ?= $(BATCH_IMAGE_TAG_BASE):$(BATCH_TAG) + NAMESPACE ?= hc4ai-operator VLLM_SIMULATOR_TAG ?= v0.6.1 export VLLM_SIMULATOR_TAG @@ -94,16 +99,24 @@ CGO_LDFLAGS := $(PYTHON_LDFLAGS) $(PYTHON_LIBS) '-L$(shell pwd)/lib' -ltokenizer # Internal variables for generic targets epp_IMAGE = $(EPP_IMAGE) sidecar_IMAGE = $(SIDECAR_IMAGE) +batch_IMAGE = $(BATCH_IMAGE) epp_NAME = epp sidecar_NAME = $(SIDECAR_NAME) +batch_NAME = batch epp_LDFLAGS = -ldflags="$(LDFLAGS)" sidecar_LDFLAGS = +batch_LDFLAGS = -ldflags="$(LDFLAGS)" epp_CGO_CFLAGS = "${CGO_CFLAGS}" sidecar_CGO_CFLAGS = +batch_CGO_CFLAGS = "${CGO_CFLAGS}" epp_CGO_LDFLAGS = "${CGO_LDFLAGS}" sidecar_CGO_LDFLAGS = +batch_CGO_LDFLAGS = "${CGO_LDFLAGS}" epp_TEST_FILES = go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ sidecar_TEST_FILES = go list ./pkg/sidecar/... +batch_TEST_FILES = go list ./... | grep -v /test/ | grep -v ./pkg/batch/ + + .PHONY: help help: ## Print help @@ -142,7 +155,7 @@ format: ## Format Go source files test: test-unit test-e2e ## Run unit tests and e2e tests .PHONY: test-unit -test-unit: test-unit-epp test-unit-sidecar +test-unit: test-unit-epp test-unit-sidecar test-unit-batch .PHONY: test-unit-% test-unit-%: download-tokenizer install-dependencies ## Run unit tests @@ -173,7 +186,7 @@ lint: check-golangci-lint check-typos ## Run lint ##@ Build .PHONY: build -build: build-epp build-sidecar ## Build the project +build: build-epp build-sidecar build-batch ## Build the project .PHONY: build-% build-%: check-go install-dependencies download-tokenizer ## Build the project @@ -183,7 +196,7 @@ build-%: check-go install-dependencies download-tokenizer ## Build the project ##@ Container Build/Push .PHONY: image-build -image-build: image-build-epp image-build-sidecar ## Build Docker image +image-build: image-build-epp image-build-sidecar image-build-batch ## Build Docker image .PHONY: image-build-% image-build-%: check-container-tool ## Build Docker image ## Build Docker image using $(CONTAINER_RUNTIME) @@ -197,7 +210,7 @@ image-build-%: check-container-tool ## Build Docker image ## Build Docker image -t $($*_IMAGE) -f Dockerfile.$* . .PHONY: image-push -image-push: image-push-epp image-push-sidecar ## Push container images to registry +image-push: image-push-epp image-push-sidecar image-push-batch ## Push container images to registry .PHONY: image-push-% image-push-%: check-container-tool ## Push container image to registry diff --git a/cmd/batch/runner/runner.go b/cmd/batch/runner/runner.go index 83dde7318..788263fc9 100644 --- a/cmd/batch/runner/runner.go +++ b/cmd/batch/runner/runner.go @@ -63,14 +63,14 @@ func (r *Runner) Run(ctx context.Context) error { } var policy batch.RequestPolicy = batch.NewRandomRobinPolicy() - var impl batch.Flow = redis.NewRedisMQFlow("localhost:6379") + var impl batch.Flow = redis.NewRedisMQFlow("localhost:16379") requestChannel := policy.MergeRequestChannels(impl.RequestChannels()).Channel for w := 1; w <= *concurrency; w++ { go batch.Worker(ctx, *endpoint, httpClient, requestChannel, impl.RetryChannel(), impl.ResultChannel()) } impl.Start(ctx) - + <-ctx.Done() return nil } diff --git a/go.mod b/go.mod index 9ed37c188..b8a04ca7e 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/onsi/gomega v1.38.2 github.com/openai/openai-go v1.12.0 github.com/prometheus/client_golang v1.23.2 + github.com/alicebob/miniredis/v2 v2.35.0 github.com/stretchr/testify v1.11.1 golang.org/x/sync v0.18.0 google.golang.org/grpc v1.76.0 @@ -93,6 +94,7 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect diff --git a/pkg/batch/GUIDE.md b/pkg/batch/GUIDE.md new file mode 100644 index 000000000..a8125f882 --- /dev/null +++ b/pkg/batch/GUIDE.md @@ -0,0 +1,7 @@ +# Batch Processor - User Guide + +The batch processor helps in workflows where you have requests that are latency tolerant. I.e., SLOs in minutes/ hours instead of seconds. + +The batch processor pulls requests from a message queue (or several MQs according to a policy), sends to the Inference Gateway (IGW) and retries if necessary (e.g., message was shedded). + +![Batch Processor - Redis architecture](/docs/images/batch_processor_redis_architecture.png "BP - Redis") \ No newline at end of file diff --git a/test/integration/redisimpl_test.go b/test/integration/redisimpl_test.go index da21d8c8c..7e27d2b89 100644 --- a/test/integration/redisimpl_test.go +++ b/test/integration/redisimpl_test.go @@ -6,17 +6,17 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/llm-d/llm-d-inference-scheduler/pkg/batch" "github.com/llm-d/llm-d-inference-scheduler/pkg/batch/redis" ) -const ( - redisURL = "localhost:6379" -) - func TestRedisImpl(t *testing.T) { + s := miniredis.RunT(t) + rAddr := s.Host() + ":" + s.Port() + ctx := context.Background() - flow := redis.NewRedisMQFlow(redisURL) + flow := redis.NewRedisMQFlow(rAddr) flow.Start(ctx) flow.RetryChannel() <- batch.RetryMessage{ From 830f3ee7d6cb644ebb5a955b9376c914071f065c Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Wed, 3 Dec 2025 21:02:52 +0000 Subject: [PATCH 4/7] adding batch deployment Signed-off-by: Shimi Bandiel --- Makefile | 4 ++- cmd/batch/main.go | 5 ++-- cmd/batch/runner/runner.go | 15 ++++++----- .../inference-gateway/deployments.yaml | 26 +++++++++++++++++++ .../inference-gateway/service-accounts.yaml | 5 ++++ scripts/kind-dev-env.sh | 19 +++++++++++++- 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 120a40dad..4858b79e4 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ TARGETOS ?= $(shell go env GOOS) TARGETARCH ?= $(shell go env GOARCH) PROJECT_NAME ?= llm-d-inference-scheduler SIDECAR_IMAGE_NAME ?= llm-d-routing-sidecar +BATCH_IMAGE_NAME ?= llm-d-routing-batch VLLM_SIMULATOR_IMAGE_NAME ?= llm-d-inference-sim SIDECAR_NAME ?= pd-sidecar IMAGE_REGISTRY ?= ghcr.io/llm-d @@ -21,7 +22,7 @@ SIDECAR_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(SIDECAR_IMAGE_NAME) export SIDECAR_IMAGE ?= $(SIDECAR_IMAGE_TAG_BASE):$(SIDECAR_TAG) BATCH_TAG ?= dev export BATCH_TAG -BATCH_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/batch +BATCH_IMAGE_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BATCH_IMAGE_NAME) export BATCH_IMAGE ?= $(BATCH_IMAGE_TAG_BASE):$(BATCH_TAG) NAMESPACE ?= hc4ai-operator @@ -444,6 +445,7 @@ env-dev-kind: ## Run under kind ($(KIND_CLUSTER_NAME)) EPP_IMAGE=$(EPP_IMAGE) \ VLLM_SIMULATOR_IMAGE=${VLLM_SIMULATOR_IMAGE} \ SIDECAR_IMAGE=${SIDECAR_IMAGE} \ + BATCH_IMAGE=${BATCH_IMAGE} \ ./scripts/kind-dev-env.sh; \ fi diff --git a/cmd/batch/main.go b/cmd/batch/main.go index 2d4d0359d..a54c39223 100644 --- a/cmd/batch/main.go +++ b/cmd/batch/main.go @@ -3,14 +3,13 @@ package main import ( "os" + batchrunner "github.com/llm-d/llm-d-inference-scheduler/cmd/batch/runner" ctrl "sigs.k8s.io/controller-runtime" - - "github.com/llm-d/llm-d-inference-scheduler/cmd/batch/runner" ) func main() { - if err := runner.NewRunner().Run(ctrl.SetupSignalHandler()); err != nil { + if err := batchrunner.NewBatchRunner().Run(ctrl.SetupSignalHandler()); err != nil { os.Exit(1) } } diff --git a/cmd/batch/runner/runner.go b/cmd/batch/runner/runner.go index 788263fc9..7a1fd2ef5 100644 --- a/cmd/batch/runner/runner.go +++ b/cmd/batch/runner/runner.go @@ -1,4 +1,4 @@ -package runner +package batchrunner import ( "context" @@ -14,21 +14,22 @@ import ( "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" ) -type Runner struct { +type BatchRunner struct { } var ( setupLog = ctrl.Log.WithName("setup") logVerbosity = flag.Int("v", logging.DEFAULT, "number for the log level verbosity") concurrency = flag.Int("concurrency", 8, "number of concurrent workers") - endpoint = flag.String("endpoint", "", "inference endpoint") + endpoint = flag.String("endpoint", "http://localhost:30080/v1/completions", "inference endpoint") + redisAddr = flag.String("redis-addr", "localhost:16379", "address of the Redis server") ) -func NewRunner() *Runner { - return &Runner{} +func NewBatchRunner() *BatchRunner { + return &BatchRunner{} } -func (r *Runner) Run(ctx context.Context) error { +func (r *BatchRunner) Run(ctx context.Context) error { opts := zap.Options{ Development: true, } @@ -63,7 +64,7 @@ func (r *Runner) Run(ctx context.Context) error { } var policy batch.RequestPolicy = batch.NewRandomRobinPolicy() - var impl batch.Flow = redis.NewRedisMQFlow("localhost:16379") + var impl batch.Flow = redis.NewRedisMQFlow(*redisAddr) requestChannel := policy.MergeRequestChannels(impl.RequestChannels()).Channel for w := 1; w <= *concurrency; w++ { go batch.Worker(ctx, *endpoint, httpClient, requestChannel, impl.RetryChannel(), impl.ResultChannel()) diff --git a/deploy/components/inference-gateway/deployments.yaml b/deploy/components/inference-gateway/deployments.yaml index cc56789a3..1dc7341eb 100644 --- a/deploy/components/inference-gateway/deployments.yaml +++ b/deploy/components/inference-gateway/deployments.yaml @@ -63,3 +63,29 @@ spec: name: epp-config - name: cache emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${BATCH_NAME} + labels: + app: ${BATCH_NAME} +spec: + replicas: 1 + selector: + matchLabels: + app: ${BATCH_NAME} + template: + metadata: + labels: + app: ${BATCH_NAME} + spec: + serviceAccountName: ${BATCH_NAME} + terminationGracePeriodSeconds: 130 + containers: + - name: batch + image: ${BATCH_IMAGE} + imagePullPolicy: IfNotPresent + args: + - --redis-addr + - "redis-service:6379" \ No newline at end of file diff --git a/deploy/components/inference-gateway/service-accounts.yaml b/deploy/components/inference-gateway/service-accounts.yaml index a92013a0a..abc74b054 100644 --- a/deploy/components/inference-gateway/service-accounts.yaml +++ b/deploy/components/inference-gateway/service-accounts.yaml @@ -2,3 +2,8 @@ apiVersion: v1 kind: ServiceAccount metadata: name: ${EPP_NAME} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ${BATCH_NAME} \ No newline at end of file diff --git a/scripts/kind-dev-env.sh b/scripts/kind-dev-env.sh index 204045b18..4a4e23376 100755 --- a/scripts/kind-dev-env.sh +++ b/scripts/kind-dev-env.sh @@ -36,6 +36,13 @@ export EPP_TAG="${EPP_TAG:-dev}" EPP_IMAGE="${EPP_IMAGE:-${IMAGE_REGISTRY}/llm-d-inference-scheduler:${EPP_TAG}}" export EPP_IMAGE +# Set a default BATCH_TAG if not provided +export BATCH_TAG="${BATCH_TAG:-dev}" + +# Set a default BATCH_IMAGE if not provided +BATCH_IMAGE="${BATCH_IMAGE:-${IMAGE_REGISTRY}/llm-d-inference-scheduler:${BATCH_TAG}}" +export BATCH_IMAGE + # Set the model name to deploy export MODEL_NAME="${MODEL_NAME:-food-review}" # Extract model family (e.g., "meta-llama" from "meta-llama/Llama-3.1-8B-Instruct") @@ -48,6 +55,9 @@ export MODEL_NAME_SAFE=$(echo "${MODEL_ID}" | tr '[:upper:]' '[:lower:]' | tr ' # Set the endpoint-picker to deploy export EPP_NAME="${EPP_NAME:-${MODEL_NAME_SAFE}-endpoint-picker}" +# Set the batch to deploy +export BATCH_NAME="${BATCH_NAME:-${MODEL_NAME_SAFE}-batch}" + # Set the default routing side car image tag export SIDECAR_TAG="${SIDECAR_TAG:-dev}" @@ -203,6 +213,13 @@ else kind --name ${CLUSTER_NAME} load docker-image ${SIDECAR_IMAGE} fi +# Load the batch image into the cluster +if [ "${CONTAINER_RUNTIME}" == "podman" ]; then + podman save ${BATCH_IMAGE} -o /dev/stdout | kind --name ${CLUSTER_NAME} load image-archive /dev/stdin +else + kind --name ${CLUSTER_NAME} load docker-image ${BATCH_IMAGE} +fi + # ------------------------------------------------------------------------------ # CRD Deployment (Gateway API + GIE) # ------------------------------------------------------------------------------ @@ -236,7 +253,7 @@ envsubst '$PRIMARY_PORT' < ${EPP_CONFIG} > ${TEMP_FILE} kubectl --context ${KUBE_CONTEXT} create configmap epp-config --from-file=epp-config.yaml=${TEMP_FILE} kustomize build --enable-helm ${KUSTOMIZE_DIR} \ - | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${VLLM_SIMULATOR_IMAGE} \ + | envsubst '${POOL_NAME} ${MODEL_NAME} ${MODEL_NAME_SAFE} ${EPP_NAME} ${EPP_IMAGE} ${BATCH_NAME} ${BATCH_IMAGE} ${VLLM_SIMULATOR_IMAGE} \ ${PD_ENABLED} ${KV_CACHE_ENABLED} ${SIDECAR_IMAGE} ${TARGET_PORTS} \ ${VLLM_REPLICA_COUNT} ${VLLM_REPLICA_COUNT_P} ${VLLM_REPLICA_COUNT_D} ${VLLM_DATA_PARALLEL_SIZE}' \ | kubectl --context ${KUBE_CONTEXT} apply -f - From 6466136df615af2dc27710c08bbf6b191bdbb8f5 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Mon, 15 Dec 2025 21:23:42 +0000 Subject: [PATCH 5/7] refarctor Signed-off-by: Shimi Bandiel --- Dockerfile.batch | 7 ++++ cmd/batch/main.go | 3 +- cmd/batch/runner/runner.go | 54 ++++++++++++++++++++++++++---- cmd/epp/main.go | 2 +- pkg/batch/api.go | 1 + pkg/batch/redis/redisimpl.go | 7 ++-- pkg/batch/worker.go | 26 ++++++++++++-- pkg/metrics/metrics.go | 32 ++++++++++++++++-- test/integration/redisimpl_test.go | 1 + 9 files changed, 118 insertions(+), 15 deletions(-) diff --git a/Dockerfile.batch b/Dockerfile.batch index be6993b65..d94fb5394 100644 --- a/Dockerfile.batch +++ b/Dockerfile.batch @@ -18,6 +18,7 @@ COPY cmd/batch/ cmd/batch/ COPY pkg/batch/ pkg/batch/ COPY pkg/metrics/ pkg/metrics/ COPY pkg/common pkg/common +COPY pkg/sidecar/version pkg/sidecar/version # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command @@ -36,4 +37,10 @@ WORKDIR / COPY --from=builder /workspace/bin/batch /app/batch USER 65532:65532 +# expose gRPC, health and metrics ports +EXPOSE 9002 +EXPOSE 9003 +EXPOSE 9090 + + ENTRYPOINT ["/app/batch"] diff --git a/cmd/batch/main.go b/cmd/batch/main.go index a54c39223..9d9e4de84 100644 --- a/cmd/batch/main.go +++ b/cmd/batch/main.go @@ -4,12 +4,13 @@ import ( "os" batchrunner "github.com/llm-d/llm-d-inference-scheduler/cmd/batch/runner" + "github.com/llm-d/llm-d-inference-scheduler/pkg/metrics" ctrl "sigs.k8s.io/controller-runtime" ) func main() { - if err := batchrunner.NewBatchRunner().Run(ctrl.SetupSignalHandler()); err != nil { + if err := batchrunner.NewBatchRunner().WithCustomCollectors(metrics.GetBatchCollectors()...).Run(ctrl.SetupSignalHandler()); err != nil { os.Exit(1) } } diff --git a/cmd/batch/runner/runner.go b/cmd/batch/runner/runner.go index 7a1fd2ef5..7cc57f5f2 100644 --- a/cmd/batch/runner/runner.go +++ b/cmd/batch/runner/runner.go @@ -3,26 +3,36 @@ package batchrunner import ( "context" "flag" + "fmt" "net/http" "github.com/llm-d/llm-d-inference-scheduler/pkg/batch" "github.com/llm-d/llm-d-inference-scheduler/pkg/batch/redis" + "github.com/llm-d/llm-d-inference-scheduler/pkg/sidecar/version" + "github.com/prometheus/client_golang/prometheus" uberzap "go.uber.org/zap" "go.uber.org/zap/zapcore" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics" + runserver "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/server" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" ) type BatchRunner struct { + customCollectors []prometheus.Collector } var ( - setupLog = ctrl.Log.WithName("setup") - logVerbosity = flag.Int("v", logging.DEFAULT, "number for the log level verbosity") - concurrency = flag.Int("concurrency", 8, "number of concurrent workers") - endpoint = flag.String("endpoint", "http://localhost:30080/v1/completions", "inference endpoint") - redisAddr = flag.String("redis-addr", "localhost:16379", "address of the Redis server") + setupLog = ctrl.Log.WithName("setup") + logVerbosity = flag.Int("v", logging.DEFAULT, "number for the log level verbosity") + concurrency = flag.Int("concurrency", 8, "number of concurrent workers") + endpoint = flag.String("endpoint", "http://localhost:30080/v1/completions", "inference endpoint") + metricsPort = flag.Int("metrics-port", runserver.DefaultMetricsPort, "The metrics port") + metricsEndpointAuth = flag.Bool("metrics-endpoint-auth", true, "Enables authentication and authorization of the metrics endpoint") ) func NewBatchRunner() *BatchRunner { @@ -59,9 +69,38 @@ func (r *BatchRunner) Run(ctx context.Context) error { }) setupLog.Info("Flags processed", "flags", flags) + // --- Get Kubernetes Config --- + cfg, err := ctrl.GetConfig() + if err != nil { + setupLog.Error(err, "Failed to get Kubernetes rest config") + return err + } + + metrics.Register(r.customCollectors...) + metrics.RecordInferenceExtensionInfo(version.CommitSHA, version.BuildRef) + // Register metrics handler. + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: fmt.Sprintf(":%d", *metricsPort), + FilterProvider: func() func(c *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) { + if *metricsEndpointAuth { + return filters.WithAuthenticationAndAuthorization + } + + return nil + }(), + } + httpClient := &http.Client{ // TODO: configure } + + msrv, _ := metricsserver.NewServer(metricsServerOptions, cfg, httpClient /* TODO: not sure about using the same one*/) + go msrv.Start(ctx) + var policy batch.RequestPolicy = batch.NewRandomRobinPolicy() var impl batch.Flow = redis.NewRedisMQFlow(*redisAddr) @@ -93,7 +132,10 @@ func initLogging(opts *zap.Options) { logger := zap.New(zap.UseFlagOptions(opts), zap.RawZapOpts(uberzap.AddCaller())) ctrl.SetLogger(logger) } - +func (r *BatchRunner) WithCustomCollectors(collectors ...prometheus.Collector) *BatchRunner { + r.customCollectors = collectors + return r +} func validateFlags() error { return nil diff --git a/cmd/epp/main.go b/cmd/epp/main.go index 1952fcf30..37ae6baa0 100644 --- a/cmd/epp/main.go +++ b/cmd/epp/main.go @@ -39,7 +39,7 @@ func main() { plugins.RegisterAllPlugins() if err := runner.NewRunner(). - WithCustomCollectors(metrics.GetCollectors()...). + WithCustomCollectors(metrics.GetEPPCollectors()...). Run(ctrl.SetupSignalHandler()); err != nil { os.Exit(1) } diff --git a/pkg/batch/api.go b/pkg/batch/api.go index 1e88fc79d..01bf553cf 100644 --- a/pkg/batch/api.go +++ b/pkg/batch/api.go @@ -14,6 +14,7 @@ type Flow interface { ResultChannel() chan ResultMessage } +// TODO: how to handle retries here? type RequestPolicy interface { MergeRequestChannels(channels []RequestChannel) RequestChannel } diff --git a/pkg/batch/redis/redisimpl.go b/pkg/batch/redis/redisimpl.go index 19a21753b..267ecd817 100644 --- a/pkg/batch/redis/redisimpl.go +++ b/pkg/batch/redis/redisimpl.go @@ -3,6 +3,7 @@ package redis import ( "context" "encoding/json" + "flag" "fmt" "strconv" @@ -16,6 +17,8 @@ import ( ) var ( + redisAddr = flag.String("redis-addr", "localhost:16379", "address of the Redis server") + // TODO: externalize requestQueueName = "batch-queue" retryQueueName = "batch-sortedset-retry" @@ -30,9 +33,9 @@ type RedisMQFlow struct { resultChannel chan batch.ResultMessage } -func NewRedisMQFlow(addr string) *RedisMQFlow { +func NewRedisMQFlow() *RedisMQFlow { rdb := redis.NewClient(&redis.Options{ - Addr: addr, + Addr: *redisAddr, // TODO: check specific version of go-redis. might require higher version. // Explicitly disable maintenance notifications diff --git a/pkg/batch/worker.go b/pkg/batch/worker.go index 54d650a4e..febe3a40d 100644 --- a/pkg/batch/worker.go +++ b/pkg/batch/worker.go @@ -29,6 +29,7 @@ func Worker(ctx context.Context, endpoint string, httpClient *http.Client, reque logger.V(logutil.DEFAULT).Info("Worker finishing.") return case msg := <-requestChannel: + metrics.BatchReqs.Inc() // Increment batch request counter TODO: here? this includes retries payloadBytes := parseAndValidateRequest(resultChannel, msg) if payloadBytes == nil { continue @@ -36,14 +37,27 @@ func Worker(ctx context.Context, endpoint string, httpClient *http.Client, reque sendInferenceRequest := func() { logger.V(logutil.DEBUG).Info("Sending inference request.") - result, err := httpClient.Post(endpoint, "application/json", bytes.NewBuffer(payloadBytes)) + request, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewBuffer(payloadBytes)) if err != nil { + metrics.FailedReqs.Inc() + resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to create request to inference: %s", err.Error())) + return + } + request.Header.Set("Content-Type", "application/json") + request.Header.Set("x-gateway-inference-objective", "food-review-2") + + result, err := httpClient.Do(request) + if err != nil { + metrics.FailedReqs.Inc() resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to send request to inference: %s", err.Error())) return } defer result.Body.Close() - // Retrying on any server-side error. Assuming shedding is included here. - if result.StatusCode >= 500 && result.StatusCode < 600 { + // Retrying on too many requests or any server-side error. + if result.StatusCode == 429 || result.StatusCode >= 500 && result.StatusCode < 600 { + if result.StatusCode == 429 { + metrics.SheddedRequests.Inc() + } retryMessage(msg, retryChannel, resultChannel) } else { payloadBytes, err := io.ReadAll(result.Body) @@ -55,9 +69,11 @@ func Worker(ctx context.Context, endpoint string, httpClient *http.Client, reque err := json.Unmarshal(payloadBytes, &resultPayload) if err != nil { // Not retrying on unmarshalling error. + metrics.FailedReqs.Inc() resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to unmarshal inference result payload: %v", err)) return } + metrics.SuccessfulReqs.Inc() resultChannel <- ResultMessage{ Id: msg.Id, Payload: resultPayload, @@ -72,17 +88,20 @@ func Worker(ctx context.Context, endpoint string, httpClient *http.Client, reque func parseAndValidateRequest(resultChannel chan ResultMessage, msg RequestMessage) []byte { deadline, err := strconv.ParseInt(msg.DeadlineUnixSec, 10, 64) if err != nil { + metrics.FailedReqs.Inc() resultChannel <- CreateErrorResultMessage(msg.Id, "Failed to parse deadline, should be in Unix seconds.") return nil } if deadline < time.Now().Unix() { + metrics.ExceededDeadlineReqs.Inc() resultChannel <- CreateDeadlineExceededResultMessage(msg.Id) return nil } payloadBytes, err := json.Marshal(msg.Payload) if err != nil { + metrics.FailedReqs.Inc() resultChannel <- CreateErrorResultMessage(msg.Id, fmt.Sprintf("Failed to marshal message's payload: %s", err.Error())) return nil } @@ -98,6 +117,7 @@ func retryMessage(msg RequestMessage, retryChannel chan RetryMessage, resultChan } secondsToDeadline := deadline - time.Now().Unix() if secondsToDeadline < 0 { + metrics.ExceededDeadlineReqs.Inc() resultChannel <- CreateDeadlineExceededResultMessage(msg.Id) } else { msg.RetryCount++ diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 1fe25beda..f300678ff 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -31,12 +31,40 @@ var ( Subsystem: SchedulerSubsystem, Name: "batch_request_retries_total", Help: "Total number of batch request retries.", }) + + BatchReqs = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_request_total", + Help: "Total number of batch requests.", + }) + ExceededDeadlineReqs = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_ßexceeded_deadline_requests_total", + Help: "Total number of batch requests that exceeded their deadline.", + }) + FailedReqs = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_failed_requests_total", + Help: "Total number of batch requests that failed.", + }) + SuccessfulReqs = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_successful_requests_total", + Help: "Total number of batch requests that succeeded.", + }) + SheddedRequests = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: SchedulerSubsystem, Name: "batch_shedded_requests_total", + Help: "Total number of batch requests that were shedded.", + }) ) // GetCollectors returns all custom collectors for the llm-d-inference-scheduler. -func GetCollectors() []prometheus.Collector { +func GetEPPCollectors() []prometheus.Collector { + return []prometheus.Collector{ + SchedulerPDDecisionCount, + } +} + +// GetCollectors returns all custom collectors for the batch processor. +func GetBatchCollectors() []prometheus.Collector { return []prometheus.Collector{ - SchedulerPDDecisionCount, Retries, + Retries, BatchReqs, ExceededDeadlineReqs, FailedReqs, SuccessfulReqs, SheddedRequests, } } diff --git a/test/integration/redisimpl_test.go b/test/integration/redisimpl_test.go index 7e27d2b89..6e547f5a4 100644 --- a/test/integration/redisimpl_test.go +++ b/test/integration/redisimpl_test.go @@ -54,3 +54,4 @@ func TestRedisImpl(t *testing.T) { } } + \ No newline at end of file From cdffb4242c08276420186a8191e20829d8f08d6a Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Mon, 15 Dec 2025 21:54:47 +0000 Subject: [PATCH 6/7] externalized Redis conn Signed-off-by: Shimi Bandiel --- pkg/batch/redis/redisimpl.go | 15 ++++++++------- test/integration/redisimpl_test.go | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/batch/redis/redisimpl.go b/pkg/batch/redis/redisimpl.go index 267ecd817..08421694d 100644 --- a/pkg/batch/redis/redisimpl.go +++ b/pkg/batch/redis/redisimpl.go @@ -17,11 +17,11 @@ import ( ) var ( - redisAddr = flag.String("redis-addr", "localhost:16379", "address of the Redis server") + redisAddr = flag.String("redis.addr", "localhost:16379", "address of the Redis server") // TODO: externalize requestQueueName = "batch-queue" - retryQueueName = "batch-sortedset-retry" + retryQueueName = flag.String("redis.retry-queue-name", "batch-sortedset-retry", "name of the Redis sorted set for retry messages") resultQueueName = "batch-queue-result" ) @@ -56,7 +56,7 @@ func NewRedisMQFlow() *RedisMQFlow { func (r *RedisMQFlow) Start(ctx context.Context) { go requestWorker(ctx, r.rdb, r.requestChannel, requestQueueName) - go addMsgToRetryWorker(ctx, r.rdb, r.retryChannel, retryQueueName) + go addMsgToRetryWorker(ctx, r.rdb, r.retryChannel, *retryQueueName) go retryWorker(ctx, r.rdb, r.requestChannel) @@ -76,6 +76,7 @@ func (r *RedisMQFlow) ResultChannel() chan batch.ResultMessage { // Listening on the results channel and responsible for writing results into Redis. func resultWorker(ctx context.Context, rdb *redis.Client, resultChannel chan batch.ResultMessage, resultsQueueName string) { + logger := log.FromContext(ctx) for { select { case <-ctx.Done(): @@ -91,8 +92,8 @@ func resultWorker(ctx context.Context, rdb *redis.Client, resultChannel chan bat } err = publishRedis(ctx, rdb, resultsQueueName, msgStr) if err != nil { - // TODO: ??? - + // Not going to retry here. Just log the error. + logger.V(logutil.DEFAULT).Error(err, "Failed to publish result message to Redis") } } } @@ -164,7 +165,7 @@ func retryWorker(ctx context.Context, rdb *redis.Client, msgChannel chan batch.R default: currentTimeSec := float64(time.Now().Unix()) - results, err := rdb.ZRangeByScore(ctx, retryQueueName, &redis.ZRangeBy{ + results, err := rdb.ZRangeByScore(ctx, *retryQueueName, &redis.ZRangeBy{ Min: "0", Max: strconv.FormatFloat(currentTimeSec, 'f', -1, 64), }).Result() @@ -178,7 +179,7 @@ func retryWorker(ctx context.Context, rdb *redis.Client, msgChannel chan batch.R fmt.Println(err) } - err = rdb.ZRem(ctx, retryQueueName, msg).Err() + err = rdb.ZRem(ctx, *retryQueueName, msg).Err() if err != nil { fmt.Println(err) diff --git a/test/integration/redisimpl_test.go b/test/integration/redisimpl_test.go index 6e547f5a4..582eba92a 100644 --- a/test/integration/redisimpl_test.go +++ b/test/integration/redisimpl_test.go @@ -2,6 +2,7 @@ package integration_test import ( "context" + "flag" "strconv" "testing" "time" @@ -16,7 +17,8 @@ func TestRedisImpl(t *testing.T) { rAddr := s.Host() + ":" + s.Port() ctx := context.Background() - flow := redis.NewRedisMQFlow(rAddr) + flag.Set("redis.addr", rAddr) + flow := redis.NewRedisMQFlow() flow.Start(ctx) flow.RetryChannel() <- batch.RetryMessage{ @@ -54,4 +56,3 @@ func TestRedisImpl(t *testing.T) { } } - \ No newline at end of file From ca300febef5e288a7e4ac15a62e4bacae3af0295 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Tue, 16 Dec 2025 18:36:12 +0000 Subject: [PATCH 7/7] externalizing values to flags Signed-off-by: Shimi Bandiel --- cmd/batch/runner/runner.go | 23 +++++++++++++++++++++-- pkg/batch/api.go | 3 ++- pkg/batch/random_robin_policy.go | 2 +- pkg/batch/redis/redisimpl.go | 12 ++++++------ pkg/batch/worker.go | 5 ++++- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/cmd/batch/runner/runner.go b/cmd/batch/runner/runner.go index 7cc57f5f2..a1c9122fb 100644 --- a/cmd/batch/runner/runner.go +++ b/cmd/batch/runner/runner.go @@ -33,6 +33,8 @@ var ( endpoint = flag.String("endpoint", "http://localhost:30080/v1/completions", "inference endpoint") metricsPort = flag.Int("metrics-port", runserver.DefaultMetricsPort, "The metrics port") metricsEndpointAuth = flag.Bool("metrics-endpoint-auth", true, "Enables authentication and authorization of the metrics endpoint") + requestMergePolicy = flag.String("request-merge-policy", "random-robin", "The request merge policy to use. Supported policies: random-robin") + messageQueueImpl = flag.String("message-queue-impl", "redis-pubsub", "The message queue implementation to use. Supported implementations: redis-pubsub") ) func NewBatchRunner() *BatchRunner { @@ -101,9 +103,26 @@ func (r *BatchRunner) Run(ctx context.Context) error { msrv, _ := metricsserver.NewServer(metricsServerOptions, cfg, httpClient /* TODO: not sure about using the same one*/) go msrv.Start(ctx) - var policy batch.RequestPolicy = batch.NewRandomRobinPolicy() + var policy batch.RequestMergePolicy + switch *requestMergePolicy { + case "random-robin": + policy = batch.NewRandomRobinPolicy() + default: + // TODO: validate this actually works + setupLog.Error(nil, "Unknown request merge policy", "policy", *requestMergePolicy) + return nil + } + + var impl batch.Flow + switch *messageQueueImpl { + case "redis-pubsub": + impl = redis.NewRedisMQFlow() + default: + // TODO: validate this actually works + setupLog.Error(nil, "Unknown message queue implementation", "impl", *messageQueueImpl) + return nil + } - var impl batch.Flow = redis.NewRedisMQFlow(*redisAddr) requestChannel := policy.MergeRequestChannels(impl.RequestChannels()).Channel for w := 1; w <= *concurrency; w++ { go batch.Worker(ctx, *endpoint, httpClient, requestChannel, impl.RetryChannel(), impl.ResultChannel()) diff --git a/pkg/batch/api.go b/pkg/batch/api.go index 01bf553cf..d6a34448a 100644 --- a/pkg/batch/api.go +++ b/pkg/batch/api.go @@ -15,7 +15,7 @@ type Flow interface { } // TODO: how to handle retries here? -type RequestPolicy interface { +type RequestMergePolicy interface { MergeRequestChannels(channels []RequestChannel) RequestChannel } @@ -26,6 +26,7 @@ type RequestMessage struct { Payload map[string]any `json:"payload"` } +// TODO: decide about metadata type RequestChannel struct { Channel chan RequestMessage Metadata map[string]any diff --git a/pkg/batch/random_robin_policy.go b/pkg/batch/random_robin_policy.go index e2497ac74..3c7fd1b45 100644 --- a/pkg/batch/random_robin_policy.go +++ b/pkg/batch/random_robin_policy.go @@ -2,7 +2,7 @@ package batch import "reflect" -func NewRandomRobinPolicy() RequestPolicy { +func NewRandomRobinPolicy() RequestMergePolicy { return &RandomRobinPolicy{} } diff --git a/pkg/batch/redis/redisimpl.go b/pkg/batch/redis/redisimpl.go index 08421694d..5e3829e57 100644 --- a/pkg/batch/redis/redisimpl.go +++ b/pkg/batch/redis/redisimpl.go @@ -17,12 +17,12 @@ import ( ) var ( - redisAddr = flag.String("redis.addr", "localhost:16379", "address of the Redis server") + redisAddr = flag.String("redis.addr", "localhost:6379", "address of the Redis server") - // TODO: externalize - requestQueueName = "batch-queue" + // TODO: support multiple request queues with metadata (for policy) + requestQueueName = flag.String("redis.request-queue-name", "batch-queue", "name of the Redis queue for request messages") retryQueueName = flag.String("redis.retry-queue-name", "batch-sortedset-retry", "name of the Redis sorted set for retry messages") - resultQueueName = "batch-queue-result" + resultQueueName = flag.String("redis.result-queue-name", "batch-queue-result", "name of the Redis queue for result messages") ) // TODO: think about what to do if Redis is down @@ -54,13 +54,13 @@ func NewRedisMQFlow() *RedisMQFlow { } func (r *RedisMQFlow) Start(ctx context.Context) { - go requestWorker(ctx, r.rdb, r.requestChannel, requestQueueName) + go requestWorker(ctx, r.rdb, r.requestChannel, *requestQueueName) go addMsgToRetryWorker(ctx, r.rdb, r.retryChannel, *retryQueueName) go retryWorker(ctx, r.rdb, r.requestChannel) - go resultWorker(ctx, r.rdb, r.resultChannel, resultQueueName) + go resultWorker(ctx, r.rdb, r.resultChannel, *resultQueueName) } func (r *RedisMQFlow) RequestChannels() []batch.RequestChannel { return []batch.RequestChannel{{Channel: r.requestChannel, Metadata: map[string]any{}}} diff --git a/pkg/batch/worker.go b/pkg/batch/worker.go index febe3a40d..7f1b25e02 100644 --- a/pkg/batch/worker.go +++ b/pkg/batch/worker.go @@ -29,7 +29,10 @@ func Worker(ctx context.Context, endpoint string, httpClient *http.Client, reque logger.V(logutil.DEFAULT).Info("Worker finishing.") return case msg := <-requestChannel: - metrics.BatchReqs.Inc() // Increment batch request counter TODO: here? this includes retries + if msg.RetryCount == 0 { + // Only count first attempt as a new request. + metrics.BatchReqs.Inc() + } payloadBytes := parseAndValidateRequest(resultChannel, msg) if payloadBytes == nil { continue