diff --git a/.buildkite/package.yml b/.buildkite/package.yml index e0405b2ef7f..ab7c500adb7 100644 --- a/.buildkite/package.yml +++ b/.buildkite/package.yml @@ -1,6 +1,8 @@ notify: - slack: "#apm-server" if: build.state == "failed" + - slack: "#observablt-bots" + if: "build.state != 'passed'" env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..be09f01d2be --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,42 @@ +version: 2 +updates: + + # GitHub actions + - package-ecosystem: "github-actions" + directory: "/" + reviewers: + - "elastic/observablt-ci" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + groups: + github-actions: + patterns: + - "*" + + # GitHub composite actions + - package-ecosystem: "github-actions" + directory: "/.github/workflows/generate-smoke-tests-list" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + reviewers: + - "elastic/observablt-ci" + groups: + github-actions: + patterns: + - "*" + - package-ecosystem: "github-actions" + directory: "/.github/workflows/setup-cluster-env" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + reviewers: + - "elastic/observablt-ci" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 01d6d721bc4..6a8055f63cc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -44,13 +44,13 @@ jobs: TF_VAR_REPO: ${{ github.repository }} GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: go-version-file: 'go.mod' - - uses: rlespinasse/github-slug-action@d1ca8ffbce40ea2371e35bca356db0acacf567d2 + - uses: rlespinasse/github-slug-action@00d4236bb38fcea6fc4aa6b1dacc7187fc8e38a0 - name: Set up env run: | @@ -67,7 +67,7 @@ jobs: echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV" fi - - uses: hashicorp/vault-action@v2.6.0 + - uses: hashicorp/vault-action@v2.7.4 env: AWS_CREDENTIALS_PATH: secret/observability-team/ci/elastic-observability-aws-account-auth BENCHMARK_CLOUD_CREDENTIALS_PATH: secret/observability-team/ci/benchmark-cloud @@ -97,7 +97,7 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.3.7 terraform_wrapper: false @@ -166,7 +166,7 @@ jobs: uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current with: message: | - Nightly benchmarks failed! SDH Duty assignee, please have a look and follow this ! + Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this ! vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} @@ -187,7 +187,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "${{ github.event_name == 'schedule' && 'Nightly succesfully executed!' || 'Benchmarks succesfully executed!' }}" + "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!" }, "accessory": { "type": "button", diff --git a/.github/workflows/bump-elastic-stack.yml b/.github/workflows/bump-elastic-stack.yml index 426c7f635f4..6680ec511db 100644 --- a/.github/workflows/bump-elastic-stack.yml +++ b/.github/workflows/bump-elastic-stack.yml @@ -26,7 +26,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.filter.outputs.matrix) }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml index c76efe51de9..da8e4009e6b 100644 --- a/.github/workflows/bump-golang.yml +++ b/.github/workflows/bump-golang.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17e48009f48..d1eee923412 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version-file: go.mod cache: true @@ -45,8 +45,8 @@ jobs: os: ['macos-latest', 'ubuntu-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version-file: go.mod cache: true @@ -57,8 +57,8 @@ jobs: system-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version-file: systemtest/go.mod cache: true diff --git a/.github/workflows/smoke-tests-schedule.yml b/.github/workflows/smoke-tests-schedule.yml index 02cacd9b60c..2ac98b2fddd 100644 --- a/.github/workflows/smoke-tests-schedule.yml +++ b/.github/workflows/smoke-tests-schedule.yml @@ -17,7 +17,7 @@ jobs: outputs: matrix: ${{ steps.generate.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: generate name: Generate matrix uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current diff --git a/.github/workflows/terraform-fmt.yml b/.github/workflows/terraform-fmt.yml index d0605b73bdf..23a3518b502 100644 --- a/.github/workflows/terraform-fmt.yml +++ b/.github/workflows/terraform-fmt.yml @@ -12,6 +12,6 @@ jobs: terraform-fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: hashicorp/setup-terraform@v2 + - uses: actions/checkout@v4 + - uses: hashicorp/setup-terraform@v3 - run: terraform fmt -recursive -check -diff diff --git a/.github/workflows/update-beats.yml b/.github/workflows/update-beats.yml index ac5213772ec..31605a85463 100644 --- a/.github/workflows/update-beats.yml +++ b/.github/workflows/update-beats.yml @@ -25,8 +25,8 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.filter.outputs.matrix) }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version-file: go.mod - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current diff --git a/.go-version b/.go-version index bae5c7f667c..20a1265cf39 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.21.3 +1.21.4 diff --git a/NOTICE.txt b/NOTICE.txt index dd389790a11..e4ed62b5f71 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v0.1.1-0.20231019084040-a0f7950e9775 +Version: v0.1.1-0.20231019120308-82201b991ae5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20231019084040-a0f7950e9775/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20231019120308-82201b991ae5/LICENSE: Apache License Version 2.0, January 2004 @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20231101145342-57df903385a9 +Version: v7.0.0-alpha2.0.20231121082709-886d078ba1f0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231101145342-57df903385a9/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231121082709-886d078ba1f0/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.7.0 +Version: v0.8.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.7.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.8.1/LICENSE.txt: Apache License Version 2.0, January 2004 diff --git a/changelogs/8.10.asciidoc b/changelogs/8.10.asciidoc index e7382633e71..574c859d40f 100644 --- a/changelogs/8.10.asciidoc +++ b/changelogs/8.10.asciidoc @@ -47,6 +47,7 @@ Fix tail-based sampling discarding low throughput and low sample rate traces {pu - Aggregated metrics now consider global labels to be part of a service's identity, and high cardinality global labels may cause services to be obscured {pull}11386[11386] - Event protobuf encoding for tail-based sampling changed (to a more efficient encoding) for event timestamp and duration {pull}11386[11386] - Use object notation for data_stream fields {pull}11311[11311] +- Removed `processor.event` and `processor.name` from the data model {pull}11028[11028] [float] ==== Bug fixes diff --git a/changelogs/8.11.asciidoc b/changelogs/8.11.asciidoc index 06cf6a43eae..e1f4696b4b3 100644 --- a/changelogs/8.11.asciidoc +++ b/changelogs/8.11.asciidoc @@ -3,8 +3,17 @@ https://github.com/elastic/apm-server/compare/8.10\...8.11[View commits] +* <> * <> +[float] +[[release-notes-8.11.1]] +=== APM version 8.11.1 + +https://github.com/elastic/apm-server/compare/v8.11.0\...v8.11.1[View commits] + +No significant changes. + [float] [[release-notes-8.11.0]] === APM version 8.11.0 diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 9652643289c..5e6155333cf 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -20,3 +20,4 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits] ==== Added - Update container base image to Ubuntu 22.04 {pull}11778[11778] - Add support for returning partial success response in OTLP input {pull}11883[11883] +- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c6cb4bba8f8..851f2b786e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-0fcb9daf-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-7521d760-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-0fcb9daf-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-7521d760-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.12.0-0fcb9daf-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.12.0-7521d760-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/docs/configure/outputs/logstash.asciidoc b/docs/configure/outputs/logstash.asciidoc index ac52c93fe21..59290980cd8 100644 --- a/docs/configure/outputs/logstash.asciidoc +++ b/docs/configure/outputs/logstash.asciidoc @@ -54,7 +54,7 @@ include::../../shared-kibana-endpoint.asciidoc[tag=shared-kibana-config] === {ls} configuration pipeline Finally, you must create a {ls} configuration pipeline that listens for incoming -APM Server connections, dedots the `data_stream.*` fields, and indexes received events into {es}. +APM Server connections and indexes received events into {es}. . Use the {logstash-ref}/plugins-inputs-elastic_agent.html[Elastic Agent input plugin] to configure {ls} to receive events from the APM Server. A minimal `input` config might look like this: @@ -68,54 +68,6 @@ input { } ---- -. Use the {logstash-ref}/plugins-filters-mutate.html[Mutate filter plugin] to set up <>. -Because the {ls} {es} output doesn't understand dotted field notation, you must use this filter to -dedot the default `data_stream.*` fields sent from APM Server to {ls}. -+ -[source,conf] ----- -filter { - mutate { - rename => { - "[data_stream.type]" => "[data_stream][type]" - "[data_stream.dataset]" => "[data_stream][dataset]" - "[data_stream.namespace]" => "[data_stream][namespace]" - } - } -} ----- -+ -.Expand to learn more -[%collapsible] -==== -**** -APM Server sends data stream information to {ls} in the following format: - -[source,json] ----- -{ - "data_stream.dataset": "apm", - "data_stream.type": "traces", - "data_stream.namespace": "default" -} ----- - -{es} expects to receive data stream information in the following format: - -[source,json] ----- -"data_stream" { - "dataset": "apm", - "type": "traces", - "dataset": "default" -} ----- - -The mutation defined above transforms what APM Server sends to {ls} into a data format that {es} understands. -This allows you to automatically route APM data to the appropriate data streams. -**** -==== - . Use the {logstash-ref}/plugins-outputs-elasticsearch.html[{es} output plugin] to send events to {es} for indexing. A minimal `output` config might look like this: + @@ -142,16 +94,6 @@ input { } } -filter { - mutate { - rename => { - "[data_stream.type]" => "[data_stream][type]" - "[data_stream.dataset]" => "[data_stream][dataset]" - "[data_stream.namespace]" => "[data_stream][namespace]" - } - } -} - output { elasticsearch { data_stream => "true" @@ -184,7 +126,7 @@ APM Server sends the following `@metadata` to {ls}: <2> The current version of APM Server. In addition to `@metadata`, APM Server provides other potentially useful fields, like the -`processor.event` field and `data_stream` fields, which can be used to conditionally operate on +`data_stream` field, which can be used to conditionally operate on {apm-guide-ref}/data-model.html[event types], namespaces, or datasets. As an example, you might want to use {ls} to route all `metric` events to the same custom metrics data stream, @@ -194,7 +136,7 @@ rather than to service-specific data streams: ---- output { if [@metadata][beat] == "apm-server" { <1> - if [processor][event] == "metric" { <2> + if [data_stream][type] == "metrics" { <2> elasticsearch { index => "%{[data_stream][type]}-custom-%{[data_stream][namespace]}" <3> action => "create" <4> diff --git a/docs/data-model.asciidoc b/docs/data-model.asciidoc index 5e6f3ad9724..20e0af115df 100644 --- a/docs/data-model.asciidoc +++ b/docs/data-model.asciidoc @@ -236,9 +236,7 @@ and {apm-go-ref-v}/metrics.html[Go runtime] metrics in the Go Agent. Infrastructure and application metrics are important sources of information when debugging production systems, which is why we've made it easy to filter metrics for specific hosts or containers in the {kib} {kibana-ref}/metrics.html[metrics overview]. -Metrics have the `processor.event` property set to `metric`. - -TIP: Most agents limit keyword fields (e.g. `processor.event`) to 1024 characters, +TIP: Most agents limit keyword fields to 1024 characters, non-keyword fields (e.g. `system.memory.total`) to 10,000 characters. Metrics are stored in metric indices. diff --git a/docs/otel-metrics.asciidoc b/docs/otel-metrics.asciidoc index b50588a4085..27fad8d0703 100644 --- a/docs/otel-metrics.asciidoc +++ b/docs/otel-metrics.asciidoc @@ -38,7 +38,7 @@ include::{tab-widget-dir}/open-kibana-widget.asciidoc[] . Open the main menu, then click *Discover*. . Select `apm-*` as your index pattern. -. Filter the data to only show documents with metrics: `processor.event: "metric"` +. Filter the data to only show documents with metrics: `[data_stream][type]: "metrics"` . Narrow your search with a known OpenTelemetry field. For example, if you have an `order_value` field, add `order_value: *` to your search to return only OpenTelemetry metrics documents. diff --git a/docs/shared-docker.asciidoc b/docs/shared-docker.asciidoc index f783b5ef108..79451dba14d 100644 --- a/docs/shared-docker.asciidoc +++ b/docs/shared-docker.asciidoc @@ -2,7 +2,7 @@ ==== Run {beatname_uc} on Docker Docker images for {beatname_uc} are available from the Elastic Docker -registry. The base image is https://hub.docker.com/_/centos/[centos:7]. +registry. The base image is https://hub.docker.com/_/ubuntu[ubuntu:22.04]. A list of all published Docker images and tags is available at https://www.docker.elastic.co[www.docker.elastic.co]. diff --git a/docs/version.asciidoc b/docs/version.asciidoc index ee00b85c6b3..5f7517f251e 100644 --- a/docs/version.asciidoc +++ b/docs/version.asciidoc @@ -1,6 +1,6 @@ // doc-branch can be: master, 8.0, 8.1, etc. :doc-branch: master -:go-version: 1.21.3 +:go-version: 1.21.4 :python: 3.7 :docker: 1.12 :docker-compose: 1.11 diff --git a/go.mod b/go.mod index b7005198969..27b519300e7 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 - github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231101145342-57df903385a9 + github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0 github.com/elastic/elastic-agent-client/v7 v7.4.0 github.com/elastic/elastic-agent-libs v0.6.2 - github.com/elastic/elastic-agent-system-metrics v0.7.0 + github.com/elastic/elastic-agent-system-metrics v0.8.1 github.com/elastic/gmux v0.3.1 github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35 github.com/elastic/go-elasticsearch/v8 v8.10.0 diff --git a/go.sum b/go.sum index 656c8270b1c..aefe06004c2 100644 --- a/go.sum +++ b/go.sum @@ -132,10 +132,10 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= -github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 h1:IPo/osxNyRxn8akP082O51YyEtrPiUjkB7yxZV57AKQ= -github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231101145342-57df903385a9 h1:fCLFX9vD1V8JV0IfMApXcnhtAbYjB3Xy/xxdcClOjlA= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231101145342-57df903385a9/go.mod h1:lHmvjDNRv+ZZUW1j0OnvunpHDG77jP//HAz70s/B1wI= +github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5 h1:URlDi0TZetSAo8KAIbaMZQhM5EG8+SDgs0jTen7WvMg= +github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0 h1:ZVQuk/GSUNf/QvlC1bnI5EqXhBTpgrisbvQPT1S7p3w= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0/go.mod h1:eIy7raRUU0xxzyz5Kz6CWyeed8l7L0TYdHxXwsJKzT8= github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= @@ -144,8 +144,8 @@ github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3 github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= -github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= -github.com/elastic/elastic-agent-system-metrics v0.7.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= +github.com/elastic/elastic-agent-system-metrics v0.8.1 h1:eg6actuLeGJlIJFotHRdlAsz/3WhX2G8E0qI301IKBA= +github.com/elastic/elastic-agent-system-metrics v0.8.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= diff --git a/internal/beater/beater.go b/internal/beater/beater.go index f6c7f5e4d53..aee736e3fa1 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -700,10 +700,15 @@ func (s *Runner) newFinalBatchProcessor( } esConfig.FlushInterval = time.Second esConfig.Config = elasticsearch.DefaultConfig() + esConfig.MaxIdleConnsPerHost = 10 if err := s.elasticsearchOutputConfig.Unpack(&esConfig); err != nil { return nil, nil, err } + if esConfig.MaxRequests != 0 { + esConfig.MaxIdleConnsPerHost = esConfig.MaxRequests + } + var flushBytes int if esConfig.FlushBytes != "" { b, err := humanize.ParseBytes(esConfig.FlushBytes) diff --git a/internal/elasticsearch/config.go b/internal/elasticsearch/config.go index 3c493e75a20..8b9e6a423eb 100644 --- a/internal/elasticsearch/config.go +++ b/internal/elasticsearch/config.go @@ -44,18 +44,19 @@ var ( // Config holds all configurable fields that are used to create a Client type Config struct { - Hosts Hosts `config:"hosts" validate:"required"` - Protocol string `config:"protocol"` - Path string `config:"path"` - ProxyURL string `config:"proxy_url"` - ProxyDisable bool `config:"proxy_disable"` - Timeout time.Duration `config:"timeout"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` - APIKey string `config:"api_key"` - Headers map[string]string `config:"headers"` - MaxRetries int `config:"max_retries"` + Hosts Hosts `config:"hosts" validate:"required"` + Protocol string `config:"protocol"` + Path string `config:"path"` + ProxyURL string `config:"proxy_url"` + ProxyDisable bool `config:"proxy_disable"` + Timeout time.Duration `config:"timeout"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` + APIKey string `config:"api_key"` + Headers map[string]string `config:"headers"` + MaxRetries int `config:"max_retries"` + MaxIdleConnsPerHost int `config:",ignore"` // CompressionLevel holds the gzip compression level used when bulk indexing // with go-docappender; it is otherwise ignored. @@ -145,9 +146,10 @@ func NewHTTPTransport(cfg *Config) (*http.Transport, error) { dialer := transport.NetDialer(cfg.Timeout) tlsDialer := transport.TLSDialer(dialer, tlsConfig, cfg.Timeout) return &http.Transport{ - Proxy: proxy, - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: tlsConfig.ToConfig(), + Proxy: proxy, + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: tlsConfig.ToConfig(), + MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost, }, nil } diff --git a/internal/elasticsearch/config_test.go b/internal/elasticsearch/config_test.go index 2676d31a6b9..3593bf02336 100644 --- a/internal/elasticsearch/config_test.go +++ b/internal/elasticsearch/config_test.go @@ -146,10 +146,11 @@ func TestBeatsConfigSynced(t *testing.T) { // TODO(simitt): take a closer look at ES ouput changes in libbeat // introduced with https://github.com/elastic/beats/pull/25219 localStructExceptions := map[string]interface{}{ - "ssl": nil, - "timeout": nil, - "proxy_disable": nil, - "proxy_url": nil, + "ssl": nil, + "timeout": nil, + "proxy_disable": nil, + "proxy_url": nil, + "maxidleconnsperhost": nil, } for name, localStructField := range localStructFields { if _, ok := localStructExceptions[name]; ok { diff --git a/packaging.mk b/packaging.mk index 36afe0b0f0d..5a72f414f27 100644 --- a/packaging.mk +++ b/packaging.mk @@ -28,13 +28,13 @@ DOCKER_BUILD_ARGS := \ DOCKER_IMAGES := \ build/docker/apm-server-$(APM_SERVER_VERSION).txt \ build/docker/apm-server-$(APM_SERVER_VERSION)-SNAPSHOT.txt \ - build/docker/apm-server-ubi9-$(APM_SERVER_VERSION).txt \ - build/docker/apm-server-ubi9-$(APM_SERVER_VERSION)-SNAPSHOT.txt + build/docker/apm-server-ubi-$(APM_SERVER_VERSION).txt \ + build/docker/apm-server-ubi-$(APM_SERVER_VERSION)-SNAPSHOT.txt build/docker/%.txt: DOCKER_IMAGE_TAG := docker.elastic.co/apm/apm-server:% build/docker/%.txt: VERSION := $(APM_SERVER_VERSION) build/docker/%-SNAPSHOT.txt: VERSION := $(APM_SERVER_VERSION)-SNAPSHOT -build/docker/apm-server-ubi9-%.txt: DOCKER_BUILD_ARGS+=--build-arg BASE_IMAGE=docker.elastic.co/ubi9/ubi-minimal +build/docker/apm-server-ubi-%.txt: DOCKER_BUILD_ARGS+=--build-arg BASE_IMAGE=docker.elastic.co/ubi9/ubi-minimal .PHONY: $(DOCKER_IMAGES) $(DOCKER_IMAGES): @@ -43,7 +43,7 @@ $(DOCKER_IMAGES): # Docker image tarballs. We distribute UBI8 Docker images only for AMD64. DOCKER_IMAGE_SUFFIX := docker-image$(if $(findstring arm64,$(GOARCH)),-arm64).tar.gz -DOCKER_IMAGE_PREFIXES := apm-server $(if $(findstring amd64,$(GOARCH)), apm-server-ubi9) +DOCKER_IMAGE_PREFIXES := apm-server $(if $(findstring amd64,$(GOARCH)), apm-server-ubi) DOCKER_IMAGE_RELEASE_TARBALLS := $(patsubst %, $(DISTDIR)/%-$(APM_SERVER_VERSION)-$(DOCKER_IMAGE_SUFFIX), $(DOCKER_IMAGE_PREFIXES)) DOCKER_IMAGE_SNAPSHOT_TARBALLS := $(patsubst %, $(DISTDIR)/%-$(APM_SERVER_VERSION)-SNAPSHOT-$(DOCKER_IMAGE_SUFFIX), $(DOCKER_IMAGE_PREFIXES)) diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index b0d0385a987..b9f9a09c165 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=ubuntu:22.04 # Build the apm-server binary. The golang image version is kept # up to date with go.mod by Makefile. -FROM golang:1.21.3 as builder +FROM golang:1.21.4 as builder WORKDIR /src COPY go.mod go.sum /src/ COPY internal/glog/go.mod /src/internal/glog/go.mod diff --git a/packaging/ironbank/Dockerfile b/packaging/ironbank/Dockerfile index 16fd92374ba..ad597e69fe0 100644 --- a/packaging/ironbank/Dockerfile +++ b/packaging/ironbank/Dockerfile @@ -4,7 +4,7 @@ ################################################################################ ARG BASE_REGISTRY=registry1.dsop.io ARG BASE_IMAGE=redhat/ubi/ubi9 -ARG BASE_TAG=9.2 +ARG BASE_TAG=9.3 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS prep_files diff --git a/packaging/ironbank/hardening_manifest.yaml b/packaging/ironbank/hardening_manifest.yaml index 427862d8a81..d456b73c335 100644 --- a/packaging/ironbank/hardening_manifest.yaml +++ b/packaging/ironbank/hardening_manifest.yaml @@ -14,7 +14,7 @@ tags: # Build args passed to Dockerfile ARGs args: BASE_IMAGE: "redhat/ubi/ubi9" - BASE_TAG: "9.2" + BASE_TAG: "9.3" ELASTIC_STACK: "${APM_SERVER_VERSION}" ELASTIC_PRODUCT: "apm-server" diff --git a/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json b/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json index ef73e0dc062..b358fe48642 100644 --- a/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json +++ b/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json @@ -8,18 +8,18 @@ }, "client": { "geo": { - "city_name": "dynamic", + "city_name": "Berlin", "continent_name": "Europe", - "country_iso_code": "ES", - "country_name": "Spain", + "country_iso_code": "DE", + "country_name": "Germany", "location": { "lat": "dynamic", "lon": "dynamic" }, - "region_iso_code": "dynamic", - "region_name": "dynamic" + "region_iso_code": "DE-BE", + "region_name": "Land Berlin" }, - "ip": "195.55.79.118" + "ip": "178.162.206.244" }, "data_stream": { "dataset": "apm.app.unknown", @@ -51,7 +51,7 @@ "name": "unknown" }, "source": { - "ip": "195.55.79.118", + "ip": "178.162.206.244", "nat": { "ip": "127.0.0.1" } diff --git a/systemtest/otlp_test.go b/systemtest/otlp_test.go index 9122c2d6e10..6a692eba1ed 100644 --- a/systemtest/otlp_test.go +++ b/systemtest/otlp_test.go @@ -464,7 +464,7 @@ func TestOTLPGRPCLogsClientIP(t *testing.T) { defer cancel() // Override local IP address to be found in "GeoLite2-City.mmdb". - md := metadata.New(map[string]string{"X-Forwarded-For": "195.55.79.118"}) + md := metadata.New(map[string]string{"X-Forwarded-For": "178.162.206.244"}) ctx = metadata.NewOutgoingContext(ctx, md) conn, err := grpc.Dial(serverAddr(srv), grpc.WithInsecure(), grpc.WithBlock(), grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip"))) @@ -478,7 +478,7 @@ func TestOTLPGRPCLogsClientIP(t *testing.T) { require.NoError(t, err) result := estest.ExpectDocs(t, systemtest.Elasticsearch, "logs-apm*", nil) - approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits, "client.geo.city_name", "client.geo.location.lat", "client.geo.location.lon", "client.geo.region_iso_code", "client.geo.region_name") + approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits, "client.geo.location.lat", "client.geo.location.lon") } func newMobileLogs(body interface{}) plog.Logs { diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index 264a132b3a9..25d61694ea0 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.12.0-0fcb9daf-SNAPSHOT + version: 8.12.0-7521d760-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index c91372a1229..a9bbf2cee2c 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.12.0-0fcb9daf-SNAPSHOT + version: 8.12.0-7521d760-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index 99c906e1571..054b634c1a4 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.12.0-0fcb9daf-SNAPSHOT + version: 8.12.0-7521d760-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch diff --git a/testing/smoke/lib.sh b/testing/smoke/lib.sh index 3a3e8eff90c..0fe19a2d731 100644 --- a/testing/smoke/lib.sh +++ b/testing/smoke/lib.sh @@ -262,15 +262,15 @@ legacy_assert_ilm() { SUCCESS=false fi local ILM_HOT_PHASE=$(echo ${ILM_PHASES} | jq '.hot') - local ILM_HOT_PHASE_MAX_SIZE=$(echo ${ILM_HOT_PHASE} | jq -r '.actions.rollover.max_primary_shard_size') + local ILM_HOT_PHASE_MAX_SIZE=$(echo ${ILM_HOT_PHASE} | jq -r '.actions.rollover.max_size') local ILM_HOT_PHASE_MAX_AGE=$(echo ${ILM_HOT_PHASE} | jq -r '.actions.rollover.max_age') if [[ "${ILM_HOT_PHASE_MAX_SIZE}" != "${EXPECTED_MAX_SIZE}" ]]; then - echo "-> Invalid ILM policy ${LEGACY_ILM_POLICY}; expected hot phase max_primary_shard_size ${EXPECTED_MAX_SIZE} got ${ILM_HOT_PHASE_MAX_SIZE}" + echo "-> Invalid ILM policy \"${LEGACY_ILM_POLICY}\"; expected hot phase \"max_age\" ${EXPECTED_MAX_SIZE} got ${ILM_HOT_PHASE_MAX_SIZE}" echo "${ILM_HOT_PHASE}" SUCCESS=false fi if [[ "${ILM_HOT_PHASE_MAX_AGE}" != "${EXPECTED_MAX_AGE}" ]]; then - echo "-> Invalid ILM policy ${LEGACY_ILM_POLICY}; expected hot phase max_age ${EXPECTED_MAX_AGE} got ${ILM_HOT_PHASE_MAX_AGE}" + echo "-> Invalid ILM policy \"${LEGACY_ILM_POLICY}\"; expected hot phase \"max_age\" ${EXPECTED_MAX_AGE} got ${ILM_HOT_PHASE_MAX_AGE}" echo "${ILM_HOT_PHASE}" SUCCESS=false fi