diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6c511293787..ed1435ddfdf 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -90,7 +90,7 @@ jobs: with: role-duration-seconds: 18000 # 5 hours - - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3 + - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4 with: export_to_environment: true secrets: |- diff --git a/.github/workflows/smoke-tests-ess.yml b/.github/workflows/smoke-tests-ess.yml index 867b1894bac..e0663b14d8a 100644 --- a/.github/workflows/smoke-tests-ess.yml +++ b/.github/workflows/smoke-tests-ess.yml @@ -59,7 +59,7 @@ jobs: - uses: elastic/oblt-actions/google/auth@v1 - - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3 + - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4 with: export_to_environment: true secrets: |- diff --git a/.github/workflows/smoke-tests-os.yml b/.github/workflows/smoke-tests-os.yml index a67871fc867..7a828ab6f27 100644 --- a/.github/workflows/smoke-tests-os.yml +++ b/.github/workflows/smoke-tests-os.yml @@ -53,7 +53,7 @@ jobs: - uses: elastic/oblt-actions/google/auth@v1 - - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3 + - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4 with: export_to_environment: true secrets: |- diff --git a/.go-version b/.go-version index da9594fd66f..013173af5e9 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.5 +1.22.6 diff --git a/NOTICE.txt b/NOTICE.txt index 0787c694659..159b23ff138 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -829,11 +829,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.15 +Version: v0.10.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.15/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.10.0/LICENSE: Apache License Version 2.0, January 2004 @@ -3310,11 +3310,11 @@ Contents of probable licence file $GOMODCACHE/github.com/modern-go/reflect2@v1.0 -------------------------------------------------------------------------------- Dependency : github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger -Version: v0.106.1 +Version: v0.107.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger@v0.106.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger@v0.107.0/LICENSE: Apache License Version 2.0, January 2004 @@ -5135,11 +5135,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/consumer -Version: v0.106.1 +Version: v0.107.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.106.1/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.107.0/LICENSE: Apache License @@ -5347,11 +5347,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/cons -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/pdata -Version: v1.12.0 +Version: v1.13.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.13.0/LICENSE: Apache License @@ -13387,11 +13387,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal -Version: v0.106.1 +Version: v0.107.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.106.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.107.0/LICENSE: Apache License Version 2.0, January 2004 @@ -16334,11 +16334,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.2/LICENS -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/semconv -Version: v0.106.1 +Version: v0.107.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.106.1/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.107.0/LICENSE: Apache License diff --git a/catalog-info.yaml b/catalog-info.yaml index 8861cd0409e..e8d14ac29b8 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -11,7 +11,7 @@ metadata: spec: type: tool - owner: group:apm-server + owner: group:obs-ds-intake-services lifecycle: production --- @@ -26,7 +26,7 @@ metadata: spec: type: buildkite-pipeline - owner: group:apm-server + owner: group:obs-ds-intake-services system: buildkite implementation: apiVersion: buildkite.elastic.dev/v1 @@ -45,7 +45,7 @@ spec: cancel_intermediate_builds: false skip_intermediate_builds: false teams: - apm-server: {} + obs-ds-intake-services: {} observablt-robots: {} everyone: access_level: READ_ONLY diff --git a/docker-compose.yml b/docker-compose.yml index 41e8bb5ae41..20c4db96e70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-f9f84acb-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-9e5932e8-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.16.0-f9f84acb-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.16.0-9e5932e8-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.16.0-f9f84acb-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.16.0-9e5932e8-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/go.mod b/go.mod index 48a75f861d4..e0ad3a59402 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/apm-data v1.9.1 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240724214231-5541971cc328 github.com/elastic/elastic-agent-client/v7 v7.15.0 - github.com/elastic/elastic-agent-libs v0.9.15 + github.com/elastic/elastic-agent-libs v0.10.0 github.com/elastic/elastic-agent-system-metrics v0.10.3 github.com/elastic/gmux v0.3.2 github.com/elastic/go-docappender/v2 v2.3.0 @@ -27,7 +27,7 @@ require ( github.com/jaegertracing/jaeger v1.60.0 github.com/libp2p/go-reuseport v0.4.0 github.com/modern-go/reflect2 v1.0.2 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/ryanuber/go-glob v1.0.0 @@ -41,8 +41,8 @@ require ( go.elastic.co/apm/module/apmotel/v2 v2.6.0 go.elastic.co/apm/v2 v2.6.0 go.elastic.co/fastjson v1.3.0 - go.opentelemetry.io/collector/consumer v0.106.1 - go.opentelemetry.io/collector/pdata v1.12.0 + go.opentelemetry.io/collector/consumer v0.107.0 + go.opentelemetry.io/collector/pdata v1.13.0 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 @@ -122,7 +122,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/planetscale/vtprotobuf v0.6.0 // indirect @@ -145,7 +145,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.elastic.co/apm/module/apmzap/v2 v2.6.0 // indirect go.elastic.co/ecszap v1.0.2 // indirect - go.opentelemetry.io/collector/semconv v0.106.1 // indirect + go.opentelemetry.io/collector/semconv v0.107.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index f60838e06e9..e25a013a5cf 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/elastic/elastic-agent-autodiscover v0.7.0 h1:FCrHXh5AZGrPlpAx8kBu/s/g github.com/elastic/elastic-agent-autodiscover v0.7.0/go.mod h1:zLf0SDdQXisVZxzXPxKXdj3Fa+H4bsu4HHbTEQImDz8= github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE= github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.9.15 h1:WCLtuErafUxczT/rXJa4Vr6mxwC8dgtqMbEq+qWGD4M= -github.com/elastic/elastic-agent-libs v0.9.15/go.mod h1:2VgYxHaeM+cCDBjiS2wbmTvzPGbnlXAtYrlcLefheS8= +github.com/elastic/elastic-agent-libs v0.10.0 h1:W7uvay0UYdLPtauXGsMD8Xfoe4qtcVWQR4icBgf/26Q= +github.com/elastic/elastic-agent-libs v0.10.0/go.mod h1:2VgYxHaeM+cCDBjiS2wbmTvzPGbnlXAtYrlcLefheS8= github.com/elastic/elastic-agent-system-metrics v0.10.3 h1:8pWdj8DeY8PBG/BA0DJalRpJWruDoP5QrIP/YKug5dE= github.com/elastic/elastic-agent-system-metrics v0.10.3/go.mod h1:3JwPa3zZJjmBYN87xwdLcFpHrUkWpR863jiYdg39sSc= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= @@ -313,10 +313,10 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1 h1:5w2vfMlgpt0nf0Z/F18dSv3achW5cTSdNaaNfsS0gVI= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.106.1/go.mod h1:gS1bYFpF6fHb12/5JNjY0liAjStZ91ts8wHZNw1hkz0= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1 h1:vVjPqLK3Hhbgxv+waOnEW1WuMFF8uQfqSWTcy+PtPew= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.106.1/go.mod h1:/UDVZCZzFlwqLlMWY2asvgX7+x9O7GQ3m9otJ0JA+/E= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0 h1:O3Oty9DKsilszc7aFb8dm34jCWXaf0la/pWV11LqGUU= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.107.0/go.mod h1:+dQdM2mmYNjZdwS5+6JWTSbZmGsnhUUxBcq2WH1bzBE= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0 h1:0TRxweGn0OFpl0KGCrjt99diVPyJ4NOCz97d/aA13VQ= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.107.0/go.mod h1:Nr/jBhS34p7SYnDUbaPRC2SW8zvMGQW/D6R20xz2F0g= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -444,12 +444,12 @@ go.elastic.co/ecszap v1.0.2 h1:iW5OGx8IiokiUzx/shD4AJCPFMC9uUtr7ycaiEIU++I= go.elastic.co/ecszap v1.0.2/go.mod h1:dJkSlK3BTiwG/qXhCwe50Mz/jwu854vSip8sIeQhNZg= go.elastic.co/fastjson v1.3.0 h1:hJO3OsYIhiqiT4Fgu0ZxAECnKASbwgiS+LMW5oCopKs= go.elastic.co/fastjson v1.3.0/go.mod h1:K9vDh7O0ODsVKV2B5e2XYLY277QZaCbB3tS1SnARvko= -go.opentelemetry.io/collector/consumer v0.106.1 h1:+AQ/Kmoc/g0WP8thwymNkXk1jeWsHDK6XyYfdezcxcc= -go.opentelemetry.io/collector/consumer v0.106.1/go.mod h1:oy6pR/v5o/N9cxsICskyt//bU8k8EG0JeOO1MTDfs5A= -go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA= -go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI= -go.opentelemetry.io/collector/semconv v0.106.1 h1:x0OSXrQCFinqZNUPTKrHU0gnbwngOVOPyhedQCDyDoQ= -go.opentelemetry.io/collector/semconv v0.106.1/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= +go.opentelemetry.io/collector/consumer v0.107.0 h1:fF/+xyv9BfXQUvuJqkljrpzKyBQExDQt6zB5rzGyuHs= +go.opentelemetry.io/collector/consumer v0.107.0/go.mod h1:wgWpFes9sbnZ11XeJPSeutU8GJx6dT/gzSUqHpaZZQA= +go.opentelemetry.io/collector/pdata v1.13.0 h1:eV3NQt2f1UcaibkziMvGTQI34LlpiYBUGp1yP0G/Cxw= +go.opentelemetry.io/collector/pdata v1.13.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI= +go.opentelemetry.io/collector/semconv v0.107.0 h1:MrrUR4L4tu3IE1JxsxtT/PxjVUqvd6SC9d/dQzk/OxA= +go.opentelemetry.io/collector/semconv v0.107.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= diff --git a/internal/beater/beater.go b/internal/beater/beater.go index efe1191e1b9..05549a5a45a 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -179,43 +179,12 @@ func (s *Runner) Run(ctx context.Context) error { } } - // Obtain the memory limit for the APM Server process. Certain config - // values will be sized according to the maximum memory set for the server. - var memLimitGB float64 - if cgroupReader := newCgroupReader(); cgroupReader != nil { - if limit, err := cgroupMemoryLimit(cgroupReader); err != nil { - s.logger.Warn(err) - } else { - memLimitGB = float64(limit) / 1024 / 1024 / 1024 - } - } - if limit, err := systemMemoryLimit(); err != nil { - s.logger.Warn(err) - } else { - var fallback bool - if memLimitGB <= 0 { - s.logger.Info("no cgroups detected, falling back to total system memory") - fallback = true - } - if memLimitGB > float64(limit) { - s.logger.Info("cgroup memory limit exceed available memory, falling back to the total system memory") - fallback = true - } - if fallback { - // If no cgroup limit is set, return a fraction of the total memory - // to have a margin of safety for other processes. The fraction value - // of 0.625 is used to keep the 80% of the total system memory limit - // to be 50% of the total for calculating the number of decoders. - memLimitGB = float64(limit) / 1024 / 1024 / 1024 * 0.625 - } - } - if memLimitGB <= 0 { - memLimitGB = 1 - s.logger.Infof( - "failed to discover memory limit, default to %0.1fgb of memory", - memLimitGB, - ) - } + memLimitGB := processMemoryLimit( + newCgroupReader(), + sysMemoryReaderFunc(systemMemoryLimit), + s.logger, + ) + if s.config.MaxConcurrentDecoders == 0 { s.config.MaxConcurrentDecoders = maxConcurrentDecoders(memLimitGB) s.logger.Infof("MaxConcurrentDecoders set to %d based on 80 percent of %0.1fgb of memory", @@ -1035,6 +1004,49 @@ func queryClusterUUID(ctx context.Context, esClient *elasticsearch.Client) error return nil } +// processMemoryLimit obtains the memory limit for the APM Server process. Certain config +// values will be sized according to the maximum memory set for the server. +func processMemoryLimit(cgroups cgroupReader, sys sysMemoryReader, logger *logp.Logger) (memLimitGB float64) { + var memLimit uint64 + if cgroups != nil { + if limit, err := cgroupMemoryLimit(cgroups); err != nil { + logger.Warn(err) + } else { + memLimit = limit + } + } + if limit, err := sys.Limit(); err != nil { + logger.Warn(err) + } else { + var fallback bool + if memLimit <= 0 { + logger.Info("no cgroups detected, falling back to total system memory") + fallback = true + } + if memLimit > limit { + logger.Info("cgroup memory limit exceed available memory, falling back to the total system memory") + fallback = true + } + if fallback { + // If no cgroup limit is set, return a fraction of the total memory + // to have a margin of safety for other processes. The fraction value + // of 0.625 is used to keep the 80% of the total system memory limit + // to be 50% of the total for calculating the number of decoders. + memLimit = uint64(float64(limit) * 0.625) + } + } + // Convert the memory limit to gigabytes to calculate the config values. + memLimitGB = float64(memLimit) / (1 << 30) + if memLimitGB <= 0 { + memLimitGB = 1 + logger.Infof( + "failed to discover memory limit, default to %0.1fgb of memory", + memLimitGB, + ) + } + return +} + type nopProcessingSupporter struct { } diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 4fec5b897f8..289ef975303 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -22,6 +22,7 @@ import ( "context" "encoding/json" "encoding/pem" + "errors" "fmt" "net/http" "net/http/httptest" @@ -40,6 +41,10 @@ import ( agentconfig "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/opt" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup/cgv1" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup/cgv2" "github.com/elastic/go-docappender/v2" ) @@ -279,3 +284,90 @@ func TestNewInstrumentation(t *testing.T) { assert.Equal(t, map[string]string{"k1": "val", "k2": "new val"}, <-labels) assert.Equal(t, "Bearer secret", auth) } + +func TestProcessMemoryLimit(t *testing.T) { + l := logp.NewLogger("test") + const gb = 1 << 30 + for name, testCase := range map[string]struct { + cgroups cgroupReader + sys sysMemoryReader + wantMemLimitGB float64 + }{ + "LimitErrShouldResultInDefaultLimit": { + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 0, errors.New("test") + }), + wantMemLimitGB: 1, + }, + "NilCgroupsShouldResultInScaledSysLimit": { + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 6.25, + }, + "CgroupsErrShouldResultInScaledSysLimit": { + cgroups: mockCgroupReader{errv: errors.New("test")}, + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 6.25, + }, + "CgroupsV1OkLimitShouldResultInCgroupsV1OkLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV1, v1: &cgroup.StatsV1{ + Memory: &cgv1.MemorySubsystem{ + Mem: cgv1.MemoryData{ + Limit: opt.Bytes{Bytes: gb}, + }, + }, + }}, + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 1, + }, + "CgroupsV1OverMaxLimitShouldResultInScaledSysLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV1, v1: &cgroup.StatsV1{ + Memory: &cgv1.MemorySubsystem{ + Mem: cgv1.MemoryData{ + Limit: opt.Bytes{Bytes: 15 * gb}, + }, + }, + }}, + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 6.25, + }, + "CgroupsV2OkLimitShouldResultInCgroupsV1OkLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV2, v2: &cgroup.StatsV2{ + Memory: &cgv2.MemorySubsystem{ + Mem: cgv2.MemoryData{ + Max: opt.BytesOpt{Bytes: opt.UintWith(gb)}, + }, + }, + }}, + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 1, + }, + "CgroupsV2OverMaxLimitShouldResultInScaledSysLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV2, v2: &cgroup.StatsV2{ + Memory: &cgv2.MemorySubsystem{ + Mem: cgv2.MemoryData{ + Max: opt.BytesOpt{Bytes: opt.UintWith(15 * gb)}, + }, + }, + }}, + sys: sysMemoryReaderFunc(func() (uint64, error) { + return 10 * gb, nil + }), + wantMemLimitGB: 6.25, + }, + } { + t.Run(name, func(t *testing.T) { + memLimitGB := processMemoryLimit(testCase.cgroups, testCase.sys, l) + assert.Equal(t, testCase.wantMemLimitGB, memLimitGB) + }) + } +} diff --git a/internal/beater/memlimit_cgroup.go b/internal/beater/memlimit_cgroup.go index 577f5c24025..08de74363e8 100644 --- a/internal/beater/memlimit_cgroup.go +++ b/internal/beater/memlimit_cgroup.go @@ -27,7 +27,16 @@ import ( "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" ) -func newCgroupReader() *cgroup.Reader { +// cgroupReader defines a short interface useful for testing purposes +// that provides a way to obtain cgroups process memory limit. +// Implemented by github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup Reader. +type cgroupReader interface { + CgroupsVersion(int) (cgroup.CgroupsVersion, error) + GetV1StatsForProcess(int) (*cgroup.StatsV1, error) + GetV2StatsForProcess(int) (*cgroup.StatsV2, error) +} + +func newCgroupReader() cgroupReader { cgroupOpts := cgroup.ReaderOptions{ RootfsMountpoint: resolve.NewTestResolver(""), IgnoreRootCgroups: true, @@ -37,13 +46,16 @@ func newCgroupReader() *cgroup.Reader { if isset { cgroupOpts.CgroupsHierarchyOverride = override } - reader, _ := cgroup.NewReaderOptions(cgroupOpts) + reader, err := cgroup.NewReaderOptions(cgroupOpts) + if err != nil { + return nil + } return reader } // Returns the cgroup maximum memory if running within a cgroup in GigaBytes, // otherwise, it returns 0 and an error. -func cgroupMemoryLimit(rdr *cgroup.Reader) (uint64, error) { +func cgroupMemoryLimit(rdr cgroupReader) (uint64, error) { pid := os.Getpid() vers, err := rdr.CgroupsVersion(pid) if err != nil { diff --git a/internal/beater/memlimit_cgroup_test.go b/internal/beater/memlimit_cgroup_test.go new file mode 100644 index 00000000000..c0c013ea523 --- /dev/null +++ b/internal/beater/memlimit_cgroup_test.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beater + +import ( + "errors" + "testing" + + "github.com/elastic/elastic-agent-libs/opt" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup/cgv1" + "github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup/cgv2" + + "github.com/stretchr/testify/assert" +) + +func TestCgroupMemoryLimit(t *testing.T) { + err := errors.New("test") + for name, testCase := range map[string]struct { + cgroups cgroupReader + wantErr bool + wantLimit uint64 + }{ + "CgroupsVersionErrShouldResultInError": { + cgroups: mockCgroupReader{errv: err}, + wantErr: true, + }, + "CgroupsInvalidVersionShouldResultInError": { + cgroups: mockCgroupReader{v: -1}, + wantErr: true, + }, + "CgroupsV1ErrShouldResultInError": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV1, errv1: err}, + wantErr: true, + }, + "CgroupsV1NilLimitShouldResultInError": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV1, v1: &cgroup.StatsV1{}}, + wantErr: true, + }, + "CgroupsV1OkLimitShouldResultInOkLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV1, v1: &cgroup.StatsV1{ + Memory: &cgv1.MemorySubsystem{ + Mem: cgv1.MemoryData{ + Limit: opt.Bytes{Bytes: 1000}, + }, + }, + }}, + wantLimit: 1000, + }, + "CgroupsV2ErrShouldResultInError": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV2, errv2: err}, + wantErr: true, + }, + "CgroupsV2NilLimitShouldResultInError": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV2, v2: &cgroup.StatsV2{}}, + wantErr: true, + }, + "CgroupsV2OkLimitShouldResultInOkLimit": { + cgroups: mockCgroupReader{v: cgroup.CgroupsV2, v2: &cgroup.StatsV2{ + Memory: &cgv2.MemorySubsystem{ + Mem: cgv2.MemoryData{ + Max: opt.BytesOpt{Bytes: opt.UintWith(1000)}, + }, + }, + }}, + wantLimit: 1000, + }, + } { + t.Run(name, func(t *testing.T) { + limit, err := cgroupMemoryLimit(testCase.cgroups) + if testCase.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, testCase.wantLimit, limit) + }) + } +} + +type mockCgroupReader struct { + v cgroup.CgroupsVersion + v1 *cgroup.StatsV1 + v2 *cgroup.StatsV2 + errv, errv1, errv2 error +} + +func (r mockCgroupReader) CgroupsVersion(int) (cgroup.CgroupsVersion, error) { + return r.v, r.errv +} + +func (r mockCgroupReader) GetV1StatsForProcess(int) (*cgroup.StatsV1, error) { + return r.v1, r.errv1 +} + +func (r mockCgroupReader) GetV2StatsForProcess(int) (*cgroup.StatsV2, error) { + return r.v2, r.errv2 +} diff --git a/internal/beater/memlimit_system.go b/internal/beater/memlimit_system.go index abb2f8e2036..b2c02d3ba52 100644 --- a/internal/beater/memlimit_system.go +++ b/internal/beater/memlimit_system.go @@ -17,9 +17,20 @@ package beater -import ( - "github.com/elastic/go-sysinfo" -) +import "github.com/elastic/go-sysinfo" + +// sysMemoryReader defines an interface useful for testing purposes +// that provides a way to obtain the total system memory limit. +type sysMemoryReader interface { + Limit() (uint64, error) +} + +// sysMemoryReaderFunc func implementation of sysMemoryReader. +type sysMemoryReaderFunc func() (uint64, error) + +func (f sysMemoryReaderFunc) Limit() (uint64, error) { + return f() +} // systemMemoryLimit returns the total system memory. func systemMemoryLimit() (uint64, error) { diff --git a/packaging/docker/Dockerfile.wolfi b/packaging/docker/Dockerfile.wolfi index 6418ac6b5a5..fd59ec38957 100644 --- a/packaging/docker/Dockerfile.wolfi +++ b/packaging/docker/Dockerfile.wolfi @@ -30,7 +30,7 @@ RUN sed -i 's/localhost:9200/elasticsearch:9200/' apm-server.yml # Build stage 1 # Copy prepared files from the previous stage and complete the image. ################################################################################ -FROM cgr.dev/chainguard/static:latest@sha256:d7518504f59dacbc90852349c0878871199cefd4bed7952d2eeb7cc3ddbe69e5 +FROM cgr.dev/chainguard/static:latest@sha256:5e9c88174a28c259c349f308dd661a6ec61ed5f8c72ecfaefb46cceb811b55a1 ARG TARGETARCH ARG BUILD_DATE ARG VERSION diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index f1c45089425..85a5f189537 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.16.0-f9f84acb-SNAPSHOT + version: 8.16.0-9e5932e8-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index 4e1910a50c2..0f070429383 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.16.0-f9f84acb-SNAPSHOT + version: 8.16.0-9e5932e8-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index d8446c7bc0a..ed2f8cc56f0 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.16.0-f9f84acb-SNAPSHOT + version: 8.16.0-9e5932e8-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch