diff --git a/cmd/router/main.go b/cmd/router/main.go index b81036ec0ac..8859d6ddad7 100644 --- a/cmd/router/main.go +++ b/cmd/router/main.go @@ -41,6 +41,8 @@ import ( var log = logf.Log.WithName("InferenceGraphRouter") func callService(serviceUrl string, input []byte, headers http.Header) ([]byte, error) { + defer timeTrack(time.Now(), "step", serviceUrl) + log.Info("Entering callService", "url", serviceUrl) req, err := http.NewRequest("POST", serviceUrl, bytes.NewBuffer(input)) for _, h := range headersToPropagate { if values, ok := headers[h]; ok { @@ -90,13 +92,13 @@ func pickupRouteByCondition(input []byte, routes []v1alpha1.InferenceStep) *v1al return nil } -func timeTrack(start time.Time, name string) { +func timeTrack(start time.Time, nodeOrStep string, name string) { elapsed := time.Since(start) - log.Info("elapsed time", "node", name, "time", elapsed) + log.Info("elapsed time", nodeOrStep, name, "time", elapsed) } func routeStep(nodeName string, graph v1alpha1.InferenceGraphSpec, input []byte, headers http.Header) ([]byte, error) { - defer timeTrack(time.Now(), nodeName) + defer timeTrack(time.Now(), "node", nodeName) currentNode := graph.Nodes[nodeName] if currentNode.RouterType == v1alpha1.Splitter { @@ -195,12 +197,16 @@ func graphHandler(w http.ResponseWriter, req *http.Request) { var ( jsonGraph = flag.String("graph-json", "", "serialized json graph def") - headersToPropagate = strings.Split(os.Getenv(constants.RouterHeadersPropagateEnvVar), ",") + headersToPropagate []string ) func main() { flag.Parse() logf.SetLogger(zap.New()) + if headersToPropagateEnvVar, ok := os.LookupEnv(constants.RouterHeadersPropagateEnvVar); ok { + log.Info("These headers will be propagated by the router to all the steps.", "headersToPropagateEnvVar", headersToPropagateEnvVar) + headersToPropagate = strings.Split(headersToPropagateEnvVar, ",") + } inferenceGraph = &v1alpha1.InferenceGraphSpec{} err := json.Unmarshal([]byte(*jsonGraph), inferenceGraph) if err != nil { diff --git a/cmd/router/main_test.go b/cmd/router/main_test.go index d1e36fb7209..ee9aa0488bc 100644 --- a/cmd/router/main_test.go +++ b/cmd/router/main_test.go @@ -9,9 +9,15 @@ import ( "knative.dev/pkg/apis" "net/http" "net/http/httptest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "testing" ) +func init() { + logf.SetLogger(zap.New()) +} + func TestSimpleModelChainer(t *testing.T) { // Start a local HTTP server model1 := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { diff --git a/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go index 6c43ea5de8c..ebb2617b544 100644 --- a/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2022 The KServe Authors. +Copyright 2023 The KServe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/serving/v1beta1/zz_generated.deepcopy.go b/pkg/apis/serving/v1beta1/zz_generated.deepcopy.go index 5b57c3d9fcc..b58bffb80e4 100644 --- a/pkg/apis/serving/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/serving/v1beta1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2022 The KServe Authors. +Copyright 2023 The KServe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,24 +17,23 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v1beta1 import ( - constants "github.com/kserve/kserve/pkg/constants" + "github.com/kserve/kserve/pkg/constants" corev1 "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - apis "knative.dev/pkg/apis" - duckv1 "knative.dev/pkg/apis/duck/v1" - v1 "knative.dev/serving/pkg/apis/serving/v1" + "k8s.io/apimachinery/pkg/runtime" + "knative.dev/pkg/apis" + "knative.dev/pkg/apis/duck/v1" + servingv1 "knative.dev/serving/pkg/apis/serving/v1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AIXExplainerSpec) DeepCopyInto(out *AIXExplainerSpec) { *out = *in in.ExplainerExtensionSpec.DeepCopyInto(&out.ExplainerExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIXExplainerSpec. @@ -51,7 +50,6 @@ func (in *AIXExplainerSpec) DeepCopy() *AIXExplainerSpec { func (in *ARTExplainerSpec) DeepCopyInto(out *ARTExplainerSpec) { *out = *in in.ExplainerExtensionSpec.DeepCopyInto(&out.ExplainerExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ARTExplainerSpec. @@ -68,7 +66,6 @@ func (in *ARTExplainerSpec) DeepCopy() *ARTExplainerSpec { func (in *AlibiExplainerSpec) DeepCopyInto(out *AlibiExplainerSpec) { *out = *in in.ExplainerExtensionSpec.DeepCopyInto(&out.ExplainerExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlibiExplainerSpec. @@ -99,7 +96,6 @@ func (in *Batcher) DeepCopyInto(out *Batcher) { *out = new(int) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Batcher. @@ -155,7 +151,6 @@ func (in *ComponentExtensionSpec) DeepCopyInto(out *ComponentExtensionSpec) { *out = new(Batcher) (*in).DeepCopyInto(*out) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentExtensionSpec. @@ -173,7 +168,7 @@ func (in *ComponentStatusSpec) DeepCopyInto(out *ComponentStatusSpec) { *out = *in if in.Traffic != nil { in, out := &in.Traffic, &out.Traffic - *out = make([]v1.TrafficTarget, len(*in)) + *out = make([]servingv1.TrafficTarget, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -195,10 +190,9 @@ func (in *ComponentStatusSpec) DeepCopyInto(out *ComponentStatusSpec) { } if in.Address != nil { in, out := &in.Address, &out.Address - *out = new(duckv1.Addressable) + *out = new(v1.Addressable) (*in).DeepCopyInto(*out) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentStatusSpec. @@ -215,7 +209,6 @@ func (in *ComponentStatusSpec) DeepCopy() *ComponentStatusSpec { func (in *CustomExplainer) DeepCopyInto(out *CustomExplainer) { *out = *in in.PodSpec.DeepCopyInto(&out.PodSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomExplainer. @@ -232,7 +225,6 @@ func (in *CustomExplainer) DeepCopy() *CustomExplainer { func (in *CustomPredictor) DeepCopyInto(out *CustomPredictor) { *out = *in in.PodSpec.DeepCopyInto(&out.PodSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomPredictor. @@ -249,7 +241,6 @@ func (in *CustomPredictor) DeepCopy() *CustomPredictor { func (in *CustomTransformer) DeepCopyInto(out *CustomTransformer) { *out = *in in.PodSpec.DeepCopyInto(&out.PodSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomTransformer. @@ -262,38 +253,6 @@ func (in *CustomTransformer) DeepCopy() *CustomTransformer { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DeployConfig) DeepCopyInto(out *DeployConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeployConfig. -func (in *DeployConfig) DeepCopy() *DeployConfig { - if in == nil { - return nil - } - out := new(DeployConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExplainerConfig) DeepCopyInto(out *ExplainerConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExplainerConfig. -func (in *ExplainerConfig) DeepCopy() *ExplainerConfig { - if in == nil { - return nil - } - out := new(ExplainerConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExplainerExtensionSpec) DeepCopyInto(out *ExplainerExtensionSpec) { *out = *in @@ -315,7 +274,6 @@ func (in *ExplainerExtensionSpec) DeepCopyInto(out *ExplainerExtensionSpec) { *out = new(StorageSpec) (*in).DeepCopyInto(*out) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExplainerExtensionSpec. @@ -348,7 +306,6 @@ func (in *ExplainerSpec) DeepCopyInto(out *ExplainerSpec) { } in.PodSpec.DeepCopyInto(&out.PodSpec) in.ComponentExtensionSpec.DeepCopyInto(&out.ComponentExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExplainerSpec. @@ -361,25 +318,6 @@ func (in *ExplainerSpec) DeepCopy() *ExplainerSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExplainersConfig) DeepCopyInto(out *ExplainersConfig) { - *out = *in - out.AlibiExplainer = in.AlibiExplainer - out.AIXExplainer = in.AIXExplainer - out.ARTExplainer = in.ARTExplainer - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExplainersConfig. -func (in *ExplainersConfig) DeepCopy() *ExplainersConfig { - if in == nil { - return nil - } - out := new(ExplainersConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FailureInfo) DeepCopyInto(out *FailureInfo) { *out = *in @@ -387,7 +325,6 @@ func (in *FailureInfo) DeepCopyInto(out *FailureInfo) { in, out := &in.Time, &out.Time *out = (*in).DeepCopy() } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailureInfo. @@ -407,7 +344,6 @@ func (in *InferenceService) DeepCopyInto(out *InferenceService) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceService. @@ -440,7 +376,6 @@ func (in *InferenceServiceList) DeepCopyInto(out *InferenceServiceList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceServiceList. @@ -475,7 +410,6 @@ func (in *InferenceServiceSpec) DeepCopyInto(out *InferenceServiceSpec) { *out = new(TransformerSpec) (*in).DeepCopyInto(*out) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceServiceSpec. @@ -494,7 +428,7 @@ func (in *InferenceServiceStatus) DeepCopyInto(out *InferenceServiceStatus) { in.Status.DeepCopyInto(&out.Status) if in.Address != nil { in, out := &in.Address, &out.Address - *out = new(duckv1.Addressable) + *out = new(v1.Addressable) (*in).DeepCopyInto(*out) } if in.URL != nil { @@ -510,7 +444,6 @@ func (in *InferenceServiceStatus) DeepCopyInto(out *InferenceServiceStatus) { } } in.ModelStatus.DeepCopyInto(&out.ModelStatus) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceServiceStatus. @@ -523,49 +456,10 @@ func (in *InferenceServiceStatus) DeepCopy() *InferenceServiceStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InferenceServicesConfig) DeepCopyInto(out *InferenceServicesConfig) { - *out = *in - out.Explainers = in.Explainers - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceServicesConfig. -func (in *InferenceServicesConfig) DeepCopy() *InferenceServicesConfig { - if in == nil { - return nil - } - out := new(InferenceServicesConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { - *out = *in - if in.IngressClassName != nil { - in, out := &in.IngressClassName, &out.IngressClassName - *out = new(string) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig. -func (in *IngressConfig) DeepCopy() *IngressConfig { - if in == nil { - return nil - } - out := new(IngressConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LightGBMSpec) DeepCopyInto(out *LightGBMSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LightGBMSpec. @@ -586,7 +480,6 @@ func (in *LoggerSpec) DeepCopyInto(out *LoggerSpec) { *out = new(string) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoggerSpec. @@ -602,7 +495,6 @@ func (in *LoggerSpec) DeepCopy() *LoggerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelCopies) DeepCopyInto(out *ModelCopies) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCopies. @@ -623,7 +515,6 @@ func (in *ModelFormat) DeepCopyInto(out *ModelFormat) { *out = new(string) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelFormat. @@ -639,7 +530,6 @@ func (in *ModelFormat) DeepCopy() *ModelFormat { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelRevisionStates) DeepCopyInto(out *ModelRevisionStates) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelRevisionStates. @@ -662,7 +552,6 @@ func (in *ModelSpec) DeepCopyInto(out *ModelSpec) { **out = **in } in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelSpec. @@ -693,7 +582,6 @@ func (in *ModelStatus) DeepCopyInto(out *ModelStatus) { *out = new(ModelCopies) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelStatus. @@ -710,7 +598,6 @@ func (in *ModelStatus) DeepCopy() *ModelStatus { func (in *ONNXRuntimeSpec) DeepCopyInto(out *ONNXRuntimeSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ONNXRuntimeSpec. @@ -727,7 +614,6 @@ func (in *ONNXRuntimeSpec) DeepCopy() *ONNXRuntimeSpec { func (in *PMMLSpec) DeepCopyInto(out *PMMLSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PMMLSpec. @@ -744,7 +630,6 @@ func (in *PMMLSpec) DeepCopy() *PMMLSpec { func (in *PaddleServerSpec) DeepCopyInto(out *PaddleServerSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaddleServerSpec. @@ -898,7 +783,6 @@ func (in *PodSpec) DeepCopyInto(out *PodSpec) { *out = new(corev1.PodOS) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpec. @@ -935,7 +819,6 @@ func (in *PredictorExtensionSpec) DeepCopyInto(out *PredictorExtensionSpec) { *out = new(StorageSpec) (*in).DeepCopyInto(*out) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PredictorExtensionSpec. @@ -1003,7 +886,6 @@ func (in *PredictorSpec) DeepCopyInto(out *PredictorSpec) { } in.PodSpec.DeepCopyInto(&out.PodSpec) in.ComponentExtensionSpec.DeepCopyInto(&out.ComponentExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PredictorSpec. @@ -1020,7 +902,6 @@ func (in *PredictorSpec) DeepCopy() *PredictorSpec { func (in *SKLearnSpec) DeepCopyInto(out *SKLearnSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SKLearnSpec. @@ -1062,7 +943,6 @@ func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { *out = new(string) **out = **in } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageSpec. @@ -1079,7 +959,6 @@ func (in *StorageSpec) DeepCopy() *StorageSpec { func (in *TFServingSpec) DeepCopyInto(out *TFServingSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TFServingSpec. @@ -1096,7 +975,6 @@ func (in *TFServingSpec) DeepCopy() *TFServingSpec { func (in *TorchServeSpec) DeepCopyInto(out *TorchServeSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TorchServeSpec. @@ -1114,7 +992,6 @@ func (in *TransformerSpec) DeepCopyInto(out *TransformerSpec) { *out = *in in.PodSpec.DeepCopyInto(&out.PodSpec) in.ComponentExtensionSpec.DeepCopyInto(&out.ComponentExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TransformerSpec. @@ -1131,7 +1008,6 @@ func (in *TransformerSpec) DeepCopy() *TransformerSpec { func (in *TritonSpec) DeepCopyInto(out *TritonSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TritonSpec. @@ -1148,7 +1024,6 @@ func (in *TritonSpec) DeepCopy() *TritonSpec { func (in *XGBoostSpec) DeepCopyInto(out *XGBoostSpec) { *out = *in in.PredictorExtensionSpec.DeepCopyInto(&out.PredictorExtensionSpec) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XGBoostSpec. diff --git a/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go b/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go index 41ec453e5a0..a5f698711c6 100644 --- a/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go +++ b/pkg/controller/v1alpha1/inferencegraph/knative_reconciler.go @@ -126,7 +126,9 @@ func createKnativeService(componentMeta metav1.ObjectMeta, graph *v1alpha1api.In annotations[autoscaling.ClassAnnotationKey] = autoscaling.KPA } - annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) + if _, ok := annotations[autoscaling.MinScaleAnnotationKey]; !ok { + annotations[autoscaling.MinScaleAnnotationKey] = fmt.Sprint(constants.DefaultMinReplicas) + } labels = utils.Filter(componentMeta.Labels, func(key string) bool { return !utils.Includes(constants.RevisionTemplateLabelDisallowedList, key)