Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions cmd/prometheus/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"github.com/alecthomas/units"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/go-logr/logr"
"github.com/grafana/regexp"
"github.com/mwitkow/go-conntrack"
"github.com/oklog/run"
Expand Down Expand Up @@ -664,8 +665,10 @@ func main() {
// Above level 6, the k8s client would log bearer tokens in clear-text.
klog.ClampLevel(6)
klog.SetLogger(log.With(logger, "component", "k8s_client_runtime"))
klogv2.ClampLevel(6)
klogv2.SetLogger(log.With(logger, "component", "k8s_client_runtime"))
klogv2.SetLogger(logr.New(&klogGokitSink{
logger: log.With(logger, "component", "k8s_client_runtime"),
maxLevel: 6,
}))

modeAppName := "Prometheus Server"
mode := "server"
Expand Down Expand Up @@ -1897,3 +1900,31 @@ func deleteStorageData(agentMode bool, dataPath string) error {
}
return nil
}

type klogGokitSink struct {
logger log.Logger
maxLevel int
}

func (s *klogGokitSink) Init(info logr.RuntimeInfo) {}
func (s *klogGokitSink) Enabled(level int) bool { return level <= s.maxLevel }
func (s *klogGokitSink) Info(level int, msg string, keysAndValues ...any) {
if level > s.maxLevel {
return
}
kvs := append([]any{"level", "info", "msg", msg}, keysAndValues...)
_ = s.logger.Log(kvs...)
}
func (s *klogGokitSink) Error(err error, msg string, keysAndValues ...any) {
kvs := append([]any{"level", "error", "msg", msg, "err", err}, keysAndValues...)
_ = s.logger.Log(kvs...)
}
func (s *klogGokitSink) WithValues(keysAndValues ...any) logr.LogSink {
return &klogGokitSink{
logger: log.With(s.logger, keysAndValues...),
maxLevel: s.maxLevel,
}
}
func (s *klogGokitSink) WithName(name string) logr.LogSink {
return s
}
Comment on lines +1904 to +1930

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The current implementation of WithName is a no-op that simply returns s. This completely discards the sub-logger names configured by client-go (e.g., reflector, informer), making it very difficult to identify which component produced a log message.

To preserve this crucial context, we should store the logger name in the klogGokitSink struct, append new names in WithName, copy the name in WithValues, and include it as a "logger" key in the log output when it is not empty. Additionally, we can optimize the slice allocations in Info and Error by pre-allocating the slice with the correct capacity.

type klogGokitSink struct {
	logger   log.Logger
	maxLevel int
	name     string
}

func (s *klogGokitSink) Init(info logr.RuntimeInfo) {}
func (s *klogGokitSink) Enabled(level int) bool { return level <= s.maxLevel }
func (s *klogGokitSink) Info(level int, msg string, keysAndValues ...any) {
	if level > s.maxLevel {
		return
	}
	kvs := make([]any, 0, 4+len(keysAndValues)+2)
	kvs = append(kvs, "level", "info", "msg", msg)
	if s.name != "" {
		kvs = append(kvs, "logger", s.name)
	}
	kvs = append(kvs, keysAndValues...)
	_ = s.logger.Log(kvs...)
}
func (s *klogGokitSink) Error(err error, msg string, keysAndValues ...any) {
	kvs := make([]any, 0, 6+len(keysAndValues)+2)
	kvs = append(kvs, "level", "error", "msg", msg, "err", err)
	if s.name != "" {
		kvs = append(kvs, "logger", s.name)
	}
	kvs = append(kvs, keysAndValues...)
	_ = s.logger.Log(kvs...)
}
func (s *klogGokitSink) WithValues(keysAndValues ...any) logr.LogSink {
	return &klogGokitSink{
		logger:   log.With(s.logger, keysAndValues...),
		maxLevel: s.maxLevel,
		name:     s.name,
	}
}
func (s *klogGokitSink) WithName(name string) logr.LogSink {
	newName := name
	if s.name != "" {
		newName = s.name + "." + name
	}
	return &klogGokitSink{
		logger:   s.logger,
		maxLevel: s.maxLevel,
		name:     newName,
	}
}

18 changes: 12 additions & 6 deletions discovery/kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -857,26 +857,32 @@ func (d *Discovery) newEndpointSlicesByNodeInformer(plw *cache.ListWatch, object
return d.mustNewSharedIndexInformer(plw, object, resyncDisabled, indexers)
}

func (d *Discovery) informerWatchErrorHandler(r *cache.Reflector, err error) {
func (d *Discovery) informerWatchErrorHandler(ctx context.Context, r *cache.Reflector, err error) {
d.metrics.failuresCount.Inc()
cache.DefaultWatchErrorHandler(r, err)
cache.DefaultWatchErrorHandler(ctx, r, err)
}

func (d *Discovery) mustNewSharedInformer(lw cache.ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration) cache.SharedInformer {
if listWatch, ok := lw.(*cache.ListWatch); ok {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}
Comment on lines +866 to +868

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent potential nil pointer dereferences or panics in unit tests or custom environments where d.client might not be fully initialized, we should defensively check that d.client is not nil before wrapping the ListWatch with cache.ToListWatcherWithWatchListSemantics.

Suggested change
if listWatch, ok := lw.(*cache.ListWatch); ok {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}
if listWatch, ok := lw.(*cache.ListWatch); ok && d.client != nil {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}

informer := cache.NewSharedInformer(lw, exampleObject, defaultEventHandlerResyncPeriod)
// Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand.
// Invoking SetWatchErrorHandlerWithContext should fail only if the informer has been started beforehand.
// Such a scenario would suggest an incorrect use of the API, thus the panic.
if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil {
if err := informer.SetWatchErrorHandlerWithContext(d.informerWatchErrorHandler); err != nil {
panic(err)
}
return informer
}

func (d *Discovery) mustNewSharedIndexInformer(lw cache.ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
if listWatch, ok := lw.(*cache.ListWatch); ok {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}
Comment on lines +879 to +881

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent potential nil pointer dereferences or panics in unit tests or custom environments where d.client might not be fully initialized, we should defensively check that d.client is not nil before wrapping the ListWatch with cache.ToListWatcherWithWatchListSemantics.

Suggested change
if listWatch, ok := lw.(*cache.ListWatch); ok {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}
if listWatch, ok := lw.(*cache.ListWatch); ok && d.client != nil {
lw = cache.ToListWatcherWithWatchListSemantics(listWatch, d.client)
}

informer := cache.NewSharedIndexInformer(lw, exampleObject, defaultEventHandlerResyncPeriod, indexers)
// Invoking SetWatchErrorHandler should fail only if the informer has been started beforehand.
// Invoking SetWatchErrorHandlerWithContext should fail only if the informer has been started beforehand.
// Such a scenario would suggest an incorrect use of the API, thus the panic.
if err := informer.SetWatchErrorHandler(d.informerWatchErrorHandler); err != nil {
if err := informer.SetWatchErrorHandlerWithContext(d.informerWatchErrorHandler); err != nil {
panic(err)
}
return informer
Expand Down
45 changes: 21 additions & 24 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/prometheus/prometheus

go 1.25.0
go 1.26.0

require (
cloud.google.com/go/auth v0.16.0
Expand Down Expand Up @@ -89,19 +89,19 @@ require (
golang.org/x/oauth2 v0.35.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.45.0
golang.org/x/time v0.11.0
golang.org/x/time v0.14.0
golang.org/x/tools v0.44.0
google.golang.org/api v0.229.0
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9
google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.30.14
k8s.io/apimachinery v0.30.14
k8s.io/client-go v0.30.14
k8s.io/api v0.36.2
k8s.io/apimachinery v0.36.2
k8s.io/client-go v0.36.2
k8s.io/klog v1.0.0
k8s.io/klog/v2 v2.130.1
k8s.io/klog/v2 v2.140.0
sigs.k8s.io/controller-runtime v0.18.7
)

Expand All @@ -125,11 +125,12 @@ require (
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
Expand All @@ -146,9 +147,8 @@ require (
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
Expand All @@ -163,7 +163,6 @@ require (
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
Expand All @@ -177,7 +176,7 @@ require (
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
Expand All @@ -188,14 +187,17 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
go.mongodb.org/mongo-driver v1.14.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/otel/metric v1.43.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.52.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.35.0 // indirect
Expand All @@ -205,28 +207,23 @@ require (
golang.org/x/tools/godoc v0.1.0-deprecated // indirect
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gotest.tools/v3 v3.0.3 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)

replace (
// NOTE(bwplotka): Latest github.com/prometheus/common this Prometheus version.
github.com/prometheus/common => github.com/prometheus/common v0.61.0

k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
// NOTE(bwplotka): This package effectively limits k8s.io/* modules to v0.32.10,
// effectively limiting the prometheus-engine to imports this fork.
// This is no longer needed in Prometheus v3.5x, so we keep k8s.io downgraded for now.
// Supporting newer means proposing v3.6.0 to github.com/simonpasquier/klog-gokit/v3
// with the new klog textlogger package.
k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.5.0
)

// Exclude linodego v1.0.0 as it is no longer published on github.
Expand Down
Loading
Loading