Merge branch 'main' into dev-docs/release-clarify-workflow

.github/workflows/benchmarks.yml

@@ -90,7 +90,7 @@ jobs:
         with:
           role-duration-seconds: 18000 # 5 hours
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-

.github/workflows/smoke-tests-ess.yml

@@ -59,7 +59,7 @@ jobs:
 
       - uses: elastic/oblt-actions/google/auth@v1
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-

.github/workflows/smoke-tests-os.yml

@@ -53,7 +53,7 @@ jobs:
 
       - uses: elastic/oblt-actions/google/auth@v1
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-

.go-version

@@ -1 +1 @@
-1.22.5
+1.22.6

NOTICE.txt

@@ -829,11 +829,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.9.15
+Version: v0.10.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.15/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.10.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -3310,11 +3310,11 @@ Contents of probable licence file $GOMODCACHE/github.com/modern-go/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger
-Version: v0.106.1
+Version: v0.107.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger@v0.106.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger@v0.107.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -5135,11 +5135,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 --------------------------------------------------------------------------------
 Dependency : go.opentelemetry.io/collector/consumer
-Version: v0.106.1
+Version: v0.107.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.106.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.107.0/LICENSE:
 
 
                                  Apache License
@@ -5347,11 +5347,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/cons
 
 --------------------------------------------------------------------------------
 Dependency : go.opentelemetry.io/collector/pdata
-Version: v1.12.0
+Version: v1.13.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.13.0/LICENSE:
 
 
                                  Apache License
@@ -13387,11 +13387,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal
-Version: v0.106.1
+Version: v0.107.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.106.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.107.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -16334,11 +16334,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/[email protected]/LICENS
 
 --------------------------------------------------------------------------------
 Dependency : go.opentelemetry.io/collector/semconv
-Version: v0.106.1
+Version: v0.107.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.106.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.107.0/LICENSE:
 
 
                                  Apache License

catalog-info.yaml

@@ -11,7 +11,7 @@ metadata:
 
 spec:
   type: tool
-  owner: group:apm-server
+  owner: group:obs-ds-intake-services
   lifecycle: production
 
 ---
@@ -26,7 +26,7 @@ metadata:
 
 spec:
   type: buildkite-pipeline
-  owner: group:apm-server
+  owner: group:obs-ds-intake-services
   system: buildkite
   implementation:
     apiVersion: buildkite.elastic.dev/v1
@@ -45,7 +45,7 @@ spec:
       cancel_intermediate_builds: false
       skip_intermediate_builds: false
       teams:
-        apm-server: {}
+        obs-ds-intake-services: {}
         observablt-robots: {}
         everyone:
           access_level: READ_ONLY

docker-compose.yml

@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-f9f84acb-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-9e5932e8-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.16.0-f9f84acb-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.16.0-9e5932e8-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.16.0-f9f84acb-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.16.0-9e5932e8-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/elastic/apm-data v1.9.1
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240724214231-5541971cc328
 	github.com/elastic/elastic-agent-client/v7 v7.15.0
-	github.com/elastic/elastic-agent-libs v0.9.15
+	github.com/elastic/elastic-agent-libs v0.10.0
 	github.com/elastic/elastic-agent-system-metrics v0.10.3
 	github.com/elastic/gmux v0.3.2
 	github.com/elastic/go-docappender/v2 v2.3.0
@@ -27,7 +27,7 @@ require (
 	github.com/jaegertracing/jaeger v1.60.0
 	github.com/libp2p/go-reuseport v0.4.0
 	github.com/modern-go/reflect2 v1.0.2
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/ryanuber/go-glob v1.0.0
@@ -41,8 +41,8 @@ require (
 	go.elastic.co/apm/module/apmotel/v2 v2.6.0
 	go.elastic.co/apm/v2 v2.6.0
 	go.elastic.co/fastjson v1.3.0
-	go.opentelemetry.io/collector/consumer v0.106.1
-	go.opentelemetry.io/collector/pdata v1.12.0
+	go.opentelemetry.io/collector/consumer v0.107.0
+	go.opentelemetry.io/collector/pdata v1.13.0
 	go.opentelemetry.io/otel v1.28.0
 	go.opentelemetry.io/otel/metric v1.28.0
 	go.opentelemetry.io/otel/sdk/metric v1.28.0
@@ -122,7 +122,7 @@ require (
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/planetscale/vtprotobuf v0.6.0 // indirect
@@ -145,7 +145,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.elastic.co/apm/module/apmzap/v2 v2.6.0 // indirect
 	go.elastic.co/ecszap v1.0.2 // indirect
-	go.opentelemetry.io/collector/semconv v0.106.1 // indirect
+	go.opentelemetry.io/collector/semconv v0.107.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

go.sum

@@ -135,8 +135,8 @@ github.com/elastic/elastic-agent-autodiscover v0.7.0 h1:FCrHXh5AZGrPlpAx8kBu/s/g
 github.com/elastic/elastic-agent-autodiscover v0.7.0/go.mod h1:zLf0SDdQXisVZxzXPxKXdj3Fa+H4bsu4HHbTEQImDz8=
 github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.9.15 h1:WCLtuErafUxczT/rXJa4Vr6mxwC8dgtqMbEq+qWGD4M=
-github.com/elastic/elastic-agent-libs v0.9.15/go.mod h1:2VgYxHaeM+cCDBjiS2wbmTvzPGbnlXAtYrlcLefheS8=
+github.com/elastic/elastic-agent-libs v0.10.0 h1:W7uvay0UYdLPtauXGsMD8Xfoe4qtcVWQR4icBgf/26Q=
+github.com/elastic/elastic-agent-libs v0.10.0/go.mod h1:2VgYxHaeM+cCDBjiS2wbmTvzPGbnlXAtYrlcLefheS8=
 github.com/elastic/elastic-agent-system-metrics v0.10.3 h1:8pWdj8DeY8PBG/BA0DJalRpJWruDoP5QrIP/YKug5dE=
 github.com/elastic/elastic-agent-system-metrics v0.10.3/go.mod h1:3JwPa3zZJjmBYN87xwdLcFpHrUkWpR863jiYdg39sSc=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
@@ -313,10 +313,10 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1 h1:5w2vfMlgpt0nf0Z/F18dSv3achW5cTSdNaaNfsS0gVI=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1/go.mod h1:gS1bYFpF6fHb12/5JNjY0liAjStZ91ts8wHZNw1hkz0=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1 h1:vVjPqLK3Hhbgxv+waOnEW1WuMFF8uQfqSWTcy+PtPew=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1/go.mod h1:/UDVZCZzFlwqLlMWY2asvgX7+x9O7GQ3m9otJ0JA+/E=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0 h1:O3Oty9DKsilszc7aFb8dm34jCWXaf0la/pWV11LqGUU=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0/go.mod h1:+dQdM2mmYNjZdwS5+6JWTSbZmGsnhUUxBcq2WH1bzBE=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0 h1:0TRxweGn0OFpl0KGCrjt99diVPyJ4NOCz97d/aA13VQ=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0/go.mod h1:Nr/jBhS34p7SYnDUbaPRC2SW8zvMGQW/D6R20xz2F0g=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -444,12 +444,12 @@ go.elastic.co/ecszap v1.0.2 h1:iW5OGx8IiokiUzx/shD4AJCPFMC9uUtr7ycaiEIU++I=
 go.elastic.co/ecszap v1.0.2/go.mod h1:dJkSlK3BTiwG/qXhCwe50Mz/jwu854vSip8sIeQhNZg=
 go.elastic.co/fastjson v1.3.0 h1:hJO3OsYIhiqiT4Fgu0ZxAECnKASbwgiS+LMW5oCopKs=
 go.elastic.co/fastjson v1.3.0/go.mod h1:K9vDh7O0ODsVKV2B5e2XYLY277QZaCbB3tS1SnARvko=
-go.opentelemetry.io/collector/consumer v0.106.1 h1:+AQ/Kmoc/g0WP8thwymNkXk1jeWsHDK6XyYfdezcxcc=
-go.opentelemetry.io/collector/consumer v0.106.1/go.mod h1:oy6pR/v5o/N9cxsICskyt//bU8k8EG0JeOO1MTDfs5A=
-go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA=
-go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI=
-go.opentelemetry.io/collector/semconv v0.106.1 h1:x0OSXrQCFinqZNUPTKrHU0gnbwngOVOPyhedQCDyDoQ=
-go.opentelemetry.io/collector/semconv v0.106.1/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw=
+go.opentelemetry.io/collector/consumer v0.107.0 h1:fF/+xyv9BfXQUvuJqkljrpzKyBQExDQt6zB5rzGyuHs=
+go.opentelemetry.io/collector/consumer v0.107.0/go.mod h1:wgWpFes9sbnZ11XeJPSeutU8GJx6dT/gzSUqHpaZZQA=
+go.opentelemetry.io/collector/pdata v1.13.0 h1:eV3NQt2f1UcaibkziMvGTQI34LlpiYBUGp1yP0G/Cxw=
+go.opentelemetry.io/collector/pdata v1.13.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI=
+go.opentelemetry.io/collector/semconv v0.107.0 h1:MrrUR4L4tu3IE1JxsxtT/PxjVUqvd6SC9d/dQzk/OxA=
+go.opentelemetry.io/collector/semconv v0.107.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw=
 go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
 go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
 go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=

internal/beater/beater.go

@@ -179,43 +179,12 @@ func (s *Runner) Run(ctx context.Context) error {
 		}
 	}
 
-	// Obtain the memory limit for the APM Server process. Certain config
-	// values will be sized according to the maximum memory set for the server.
-	var memLimitGB float64
-	if cgroupReader := newCgroupReader(); cgroupReader != nil {
-		if limit, err := cgroupMemoryLimit(cgroupReader); err != nil {
-			s.logger.Warn(err)
-		} else {
-			memLimitGB = float64(limit) / 1024 / 1024 / 1024
-		}
-	}
-	if limit, err := systemMemoryLimit(); err != nil {
-		s.logger.Warn(err)
-	} else {
-		var fallback bool
-		if memLimitGB <= 0 {
-			s.logger.Info("no cgroups detected, falling back to total system memory")
-			fallback = true
-		}
-		if memLimitGB > float64(limit) {
-			s.logger.Info("cgroup memory limit exceed available memory, falling back to the total system memory")
-			fallback = true
-		}
-		if fallback {
-			// If no cgroup limit is set, return a fraction of the total memory
-			// to have a margin of safety for other processes. The fraction value
-			// of 0.625 is used to keep the 80% of the total system memory limit
-			// to be 50% of the total for calculating the number of decoders.
-			memLimitGB = float64(limit) / 1024 / 1024 / 1024 * 0.625
-		}
-	}
-	if memLimitGB <= 0 {
-		memLimitGB = 1
-		s.logger.Infof(
-			"failed to discover memory limit, default to %0.1fgb of memory",
-			memLimitGB,
-		)
-	}
+	memLimitGB := processMemoryLimit(
+		newCgroupReader(),
+		sysMemoryReaderFunc(systemMemoryLimit),
+		s.logger,
+	)
+
 	if s.config.MaxConcurrentDecoders == 0 {
 		s.config.MaxConcurrentDecoders = maxConcurrentDecoders(memLimitGB)
 		s.logger.Infof("MaxConcurrentDecoders set to %d based on 80 percent of %0.1fgb of memory",
@@ -1035,6 +1004,49 @@ func queryClusterUUID(ctx context.Context, esClient *elasticsearch.Client) error
 	return nil
 }
 
+// processMemoryLimit obtains the memory limit for the APM Server process. Certain config
+// values will be sized according to the maximum memory set for the server.
+func processMemoryLimit(cgroups cgroupReader, sys sysMemoryReader, logger *logp.Logger) (memLimitGB float64) {
+	var memLimit uint64
+	if cgroups != nil {
+		if limit, err := cgroupMemoryLimit(cgroups); err != nil {
+			logger.Warn(err)
+		} else {
+			memLimit = limit
+		}
+	}
+	if limit, err := sys.Limit(); err != nil {
+		logger.Warn(err)
+	} else {
+		var fallback bool
+		if memLimit <= 0 {
+			logger.Info("no cgroups detected, falling back to total system memory")
+			fallback = true
+		}
+		if memLimit > limit {
+			logger.Info("cgroup memory limit exceed available memory, falling back to the total system memory")
+			fallback = true
+		}
+		if fallback {
+			// If no cgroup limit is set, return a fraction of the total memory
+			// to have a margin of safety for other processes. The fraction value
+			// of 0.625 is used to keep the 80% of the total system memory limit
+			// to be 50% of the total for calculating the number of decoders.
+			memLimit = uint64(float64(limit) * 0.625)
+		}
+	}
+	// Convert the memory limit to gigabytes to calculate the config values.
+	memLimitGB = float64(memLimit) / (1 << 30)
+	if memLimitGB <= 0 {
+		memLimitGB = 1
+		logger.Infof(
+			"failed to discover memory limit, default to %0.1fgb of memory",
+			memLimitGB,
+		)
+	}
+	return
+}
+
 type nopProcessingSupporter struct {
 }