Skip to content

Commit dabc010

Browse files
sallyomsallyom
committed
opentelemetry tracing instrumentation
Signed-off-by: sallyom <[email protected]>
1 parent 689f6c7 commit dabc010

File tree

7 files changed

+236
-16
lines changed

7 files changed

+236
-16
lines changed

‎cmd/llm-d-routing-sidecar/main.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"github.com/llm-d/llm-d-routing-sidecar/internal/proxy"
2727
"github.com/llm-d/llm-d-routing-sidecar/internal/signals"
28+
"github.com/llm-d/llm-d-routing-sidecar/internal/tracing"
2829
)
2930

3031
func main() {
@@ -53,14 +54,26 @@ func main() {
5354
ctx := signals.SetupSignalHandler(context.Background())
5455
logger := klog.FromContext(ctx)
5556

57+
tracingConfig := tracing.NewConfigFromEnv()
58+
if tracingShutdown, err := tracing.Initialize(ctx, tracingConfig); err != nil {
59+
logger.Error(err, "failed to setup tracing, continuing with no-op tracer")
60+
} else {
61+
defer tracingShutdown()
62+
if tracingConfig.Enabled {
63+
logger.Info("tracing initialized successfully", "endpoint", tracingConfig.ExporterEndpoint, "samplingRate", tracingConfig.SamplingRate)
64+
} else {
65+
logger.Info("tracing disabled")
66+
}
67+
}
68+
5669
if *connector != proxy.ConnectorNIXLV1 && *connector != proxy.ConnectorNIXLV2 && *connector != proxy.ConnectorLMCache {
5770
logger.Info("Error: --connector must either be 'nixl', 'nixlv2' or 'lmcache'")
5871
return
5972
}
6073
if *connector == proxy.ConnectorNIXLV1 {
6174
logger.Info("Warning: nixl connector is deprecated and will be removed in a future release in favor of --connector=nixlv2")
6275
}
63-
logger.Info("p/d connector validated", "connector", connector)
76+
logger.Info("p/d connector validated", "connector", *connector)
6477

6578
// Determine namespace and pool name for SSRF protection
6679
if *enableSSRFProtection {

‎go.mod‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,40 +8,54 @@ require (
88
github.com/hashicorp/golang-lru/v2 v2.0.7
99
github.com/onsi/ginkgo/v2 v2.23.4
1010
github.com/onsi/gomega v1.37.0
11+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
12+
go.opentelemetry.io/otel v1.36.0
13+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0
14+
go.opentelemetry.io/otel/sdk v1.36.0
15+
go.opentelemetry.io/otel/trace v1.36.0
1116
k8s.io/apimachinery v0.31.3
1217
k8s.io/client-go v0.31.3
1318
k8s.io/klog/v2 v2.130.1
1419
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
1520
)
1621

1722
require (
23+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
1824
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
1925
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
26+
github.com/felixge/httpsnoop v1.0.4 // indirect
2027
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
28+
github.com/go-logr/stdr v1.2.2 // indirect
2129
github.com/go-openapi/jsonreference v0.21.0 // indirect
2230
github.com/go-openapi/swag v0.23.0 // indirect
2331
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
2432
github.com/gogo/protobuf v1.3.2 // indirect
2533
github.com/google/go-cmp v0.7.0 // indirect
2634
github.com/google/gofuzz v1.2.0 // indirect
2735
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
36+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
2837
github.com/imdario/mergo v0.3.6 // indirect
2938
github.com/json-iterator/go v1.1.12 // indirect
3039
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
3140
github.com/modern-go/reflect2 v1.0.2 // indirect
32-
github.com/rogpeppe/go-internal v1.13.1 // indirect
3341
github.com/spf13/pflag v1.0.5 // indirect
34-
github.com/stretchr/testify v1.10.0 // indirect
3542
github.com/x448/float16 v0.8.4 // indirect
43+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
44+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect
45+
go.opentelemetry.io/otel/metric v1.36.0 // indirect
46+
go.opentelemetry.io/proto/otlp v1.4.0 // indirect
3647
go.uber.org/automaxprocs v1.6.0 // indirect
3748
golang.org/x/net v0.38.0 // indirect
3849
golang.org/x/oauth2 v0.27.0 // indirect
39-
golang.org/x/sys v0.32.0 // indirect
50+
golang.org/x/sys v0.33.0 // indirect
4051
golang.org/x/term v0.30.0 // indirect
4152
golang.org/x/text v0.23.0 // indirect
4253
golang.org/x/time v0.5.0 // indirect
4354
golang.org/x/tools v0.31.0 // indirect
44-
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
55+
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
56+
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
57+
google.golang.org/grpc v1.68.1 // indirect
58+
google.golang.org/protobuf v1.36.5 // indirect
4559
gopkg.in/inf.v0 v0.9.1 // indirect
4660
gopkg.in/yaml.v2 v2.4.0 // indirect
4761
gopkg.in/yaml.v3 v3.0.1 // indirect

‎go.sum‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,20 @@
1+
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
2+
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
13
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
24
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
35
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
46
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57
github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
68
github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
9+
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
10+
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
711
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
812
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
13+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
914
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
1015
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
16+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
17+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
1118
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
1219
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
1320
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
@@ -32,6 +39,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J
3239
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
3340
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
3441
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
42+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE=
43+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI=
3544
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
3645
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
3746
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
@@ -42,11 +51,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
4251
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
4352
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
4453
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
45-
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
4654
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
4755
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
48-
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
49-
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
5056
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
5157
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
5258
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -79,8 +85,28 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
7985
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
8086
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
8187
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
88+
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
89+
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
90+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
91+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
92+
go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
93+
go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
94+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA=
95+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI=
96+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM=
97+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA=
98+
go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
99+
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
100+
go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
101+
go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
102+
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
103+
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
104+
go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
105+
go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
82106
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
83107
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
108+
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
109+
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
84110
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
85111
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
86112
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -100,8 +126,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
100126
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
101127
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
102128
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
103-
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
104-
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
129+
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
130+
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
105131
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
106132
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
107133
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -120,6 +146,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
120146
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
121147
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
122148
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
149+
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
150+
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
151+
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY=
152+
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=
153+
google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0=
154+
google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw=
123155
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
124156
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
125157
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎internal/proxy/chat_completions.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package proxy
1818

1919
import (
2020
"net/http"
21+
22+
"github.com/llm-d/llm-d-routing-sidecar/internal/tracing"
23+
"go.opentelemetry.io/otel/attribute"
2124
)
2225

2326
var (
@@ -29,15 +32,24 @@ var (
2932
)
3033

3134
func (s *Server) chatCompletionsHandler(w http.ResponseWriter, r *http.Request) {
32-
prefillPodHostPort := r.Header.Get(requestHeaderPrefillHostPort)
35+
ctx, span := tracing.StartSpan(r.Context(), "routing_proxy.request")
36+
defer span.End()
37+
38+
// Basic attributes only
39+
span.SetAttributes(
40+
attribute.String("llm_d.proxy.connector", s.connector),
41+
)
3342

43+
prefillPodHostPort := r.Header.Get(requestHeaderPrefillHostPort)
3444
if prefillPodHostPort == "" {
3545
// backward compatible behavior: to remove in next release
3646
prefillPodHostPort = r.Header.Get(requestHeaderPrefillURL)
3747
}
3848

3949
if prefillPodHostPort == "" {
4050
s.logger.V(4).Info("skip disaggregated prefill")
51+
// Update the request context for downstream handlers
52+
r = r.WithContext(ctx)
4153
s.decoderProxy.ServeHTTP(w, r)
4254
return
4355
}
@@ -54,5 +66,7 @@ func (s *Server) chatCompletionsHandler(w http.ResponseWriter, r *http.Request)
5466
}
5567

5668
s.logger.V(4).Info("SSRF protection: prefill target allowed", "target", prefillPodHostPort)
69+
70+
r = r.WithContext(ctx)
5771
s.runConnectorProtocol(w, r, prefillPodHostPort)
5872
}

‎internal/proxy/connector_nixlv2.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
5959
// Prefill Stage
6060

6161
// 1. Prepare prefill request
62-
ctx := r.Context()
63-
preq := r.Clone(ctx)
62+
preq := r.Clone(r.Context())
6463

6564
preq.Header.Add(requestHeaderRequestID, uuidStr)
6665

@@ -131,7 +130,7 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
131130
// Decode Stage
132131

133132
// 1. Prepare decode request
134-
dreq := r.Clone(ctx)
133+
dreq := r.Clone(r.Context())
135134

136135
dreq.Header.Add(requestHeaderRequestID, uuidStr)
137136

@@ -159,7 +158,6 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
159158
dreq.ContentLength = int64(len(dbody))
160159

161160
// 2. Forward to local decoder.
162-
163161
s.logger.V(5).Info("sending request to decoder", "body", string(dbody))
164162
s.decoderProxy.ServeHTTP(w, dreq)
165163
}

‎internal/proxy/proxy.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131

3232
"github.com/go-logr/logr"
3333
lru "github.com/hashicorp/golang-lru/v2"
34+
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
3435
"k8s.io/klog/v2"
3536
)
3637

@@ -102,6 +103,7 @@ type Server struct {
102103
runConnectorProtocol protocolRunner // the handler for running the protocol
103104
prefillerURLPrefix string
104105
allowlistValidator *AllowlistValidator // SSRF protection validator
106+
connector string // the connector type for tracing
105107

106108
prefillerProxies *lru.Cache[string, http.Handler] // cached prefiller proxy handlers
107109

@@ -124,6 +126,7 @@ func NewProxy(port string, decodeURL *url.URL, config Config) (*Server, error) {
124126
prefillerProxies: cache,
125127
prefillerURLPrefix: "http://",
126128
allowlistValidator: validator,
129+
connector: config.Connector,
127130
config: config,
128131
}
129132
switch config.Connector {
@@ -165,8 +168,11 @@ func (s *Server) Start(ctx context.Context) error {
165168
// Configure handlers
166169
mux := s.createRoutes()
167170

171+
// Wrap the server with OpenTelemetry HTTP instrumentation
172+
handler := otelhttp.NewHandler(mux, "routing-proxy-server")
173+
168174
server := &http.Server{
169-
Handler: mux,
175+
Handler: handler,
170176
// No ReadTimeout/WriteTimeout for LLM inference - can take hours for large contexts
171177
IdleTimeout: 300 * time.Second, // 5 minutes for keep-alive connections
172178
ReadHeaderTimeout: 30 * time.Second, // Reasonable for headers only

0 commit comments

Comments
 (0)