Merge branch 'main' into update-known-issues

.buildkite/hooks/pre-command

@@ -70,7 +70,7 @@ if command -v docker &>/dev/null; then
     DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field username "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_REGISTRY_SECRET=$(retry 5 vault kv get -field registry "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY_SECRET}"
+    retry 4 docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY_SECRET}"
     unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
     export DOCKER_REGISTRY_SECRET
     retry 4 docker pull --quiet docker.elastic.co/infra/release-manager:latest
@@ -80,7 +80,7 @@ if command -v docker &>/dev/null; then
     DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field username "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKERHUB_REGISTRY_SECRET=$(retry 5 vault kv get -field registry "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKERHUB_REGISTRY_SECRET}"
+    retry 4 docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKERHUB_REGISTRY_SECRET}"
     unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
 fi
 

.ci/update-beats.yml

@@ -44,15 +44,14 @@ sources:
           captureindex: 1
 
 conditions:
-  is:
+  is-already-updated:
     name: Is version 'github.com/elastic/beats@{{ source "beats" }}' not updated in 'go.mod'?
-    kind: file
+    kind: shell
     disablesourceinput: true
     scmid: default
     spec:
-      file: go.mod
-      matchpattern: 'github\.com/elastic/beats.*-{{ source "beats" }}'
-    failwhen: true
+      command: grep {{ source "beats" }} go.mod && exit 1 || exit 0
+    failwhen: false
 
 targets:
   beats:
@@ -64,5 +63,4 @@ targets:
       command: .ci/scripts/update-beats.sh
       environments:
         - name: PATH
-        - name: GOPATH
         - name: HOME

.github/workflows/benchmarks.yml

@@ -134,7 +134,7 @@ jobs:
           $PNG_REPORT_FILE
 
       - name: Upload PNG
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: kibana-png-report
           path: ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }}
@@ -150,7 +150,7 @@ jobs:
           echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT"
 
       - name: Upload benchmark result
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: benchmark-result

.github/workflows/ci.yml

@@ -71,7 +71,7 @@ jobs:
           go run -modfile=tools/go.mod gotest.tools/gotestsum \
           --no-color -f standard-quiet --jsonfile "$OUTPUT_JSON_FILE" --junitfile "$OUTPUT_JUNIT_FILE" \
           --raw-command -- make system-test
-      - uses: actions/upload-artifact@v3  # upload test results
+      - uses: actions/upload-artifact@v4  # upload test results
         if: ${{ always() }} # run this step even if previous step failed
         with:
           name: test-results

.github/workflows/opentelemetry.yml

@@ -3,13 +3,17 @@ name: OpenTelemetry Export Trace
 on:
   workflow_run:
     workflows:
+      - benchmarks
       - bump-elastic-stack-snapshot
       - bump-golang
       - ci
       - microbenchmark
-      - system-test-reporter
+      - run-minor-release
+      - run-patch-release
+      - smoke-tests
       - smoke-tests-os
       - smoke-tests-ess
+      - system-test-reporter
       - Terraform Format
       - update-beats
     types: [completed]

.github/workflows/run-minor-release.yml

@@ -0,0 +1,22 @@
+---
+name: run-minor-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version (semver format: major.minor.patch)'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  SLACK_CHANNEL: "#apm-server-test-release"
+
+jobs:
+  run-minor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4

.github/workflows/run-patch-release.yml

@@ -0,0 +1,22 @@
+---
+name: run-patch-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version (semver format: major.minor.patch)'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  SLACK_CHANNEL: "#apm-server-test-release"
+
+jobs:
+  run-patch:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4

.go-version

@@ -1 +1 @@
-1.21.4
+1.21.5

CHANGELOG.asciidoc

@@ -1,4 +1,22 @@
+// tag::list[]
+* <<release-notes-8.12>>
+* <<release-notes-8.11>>
+* <<release-notes-8.10>>
+* <<release-notes-8.9>>
+* <<release-notes-8.8>>
+* <<release-notes-8.7>>
+* <<release-notes-8.6>>
+* <<release-notes-8.5>>
+* <<release-notes-8.4>>
+* <<release-notes-8.3>>
+* <<release-notes-8.2>>
+* <<release-notes-8.1>>
+* <<release-notes-8.0>>
+// end::list[]
+
+// tag::includes[]
 include::./changelogs/head.asciidoc[]
+include::./changelogs/8.12.asciidoc[]
 include::./changelogs/8.11.asciidoc[]
 include::./changelogs/8.10.asciidoc[]
 include::./changelogs/8.9.asciidoc[]
@@ -11,3 +29,4 @@ include::./changelogs/8.3.asciidoc[]
 include::./changelogs/8.2.asciidoc[]
 include::./changelogs/8.1.asciidoc[]
 include::./changelogs/8.0.asciidoc[]
+// end::includes[]

NOTICE.txt

@@ -362,11 +362,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/apm-data
-Version: v0.1.1-0.20231019120308-82201b991ae5
+Version: v0.1.1-0.20231212041654-b2a4dabeb6e3
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20231019120308-82201b991ae5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20231212041654-b2a4dabeb6e3/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20231204124921-be42a3369f94
+Version: v7.0.0-alpha2.0.20231227085400-8c4a40fdbfda
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231204124921-be42a3369f94/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231227085400-8c4a40fdbfda/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -596,11 +596,11 @@ License Version 2.0.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.5.0
+Version: v7.6.0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.5.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.6.0/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 
@@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.9.0
+Version: v0.9.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -1884,11 +1884,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.11.1
+Version: v1.11.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE.txt:
 
 
                                  Apache License
@@ -6640,11 +6640,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.17.0
+Version: v0.19.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.19.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -6714,11 +6714,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
-Version: v0.13.0
+Version: v0.15.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.15.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -18332,11 +18332,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.14.0
+Version: v0.16.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.16.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -18369,11 +18369,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/exp
-Version: v0.0.0-20230713183714-613f0c0eb8a1
+Version: v0.0.0-20231127185646-65229373498e
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20230713183714-613f0c0eb8a1/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20231127185646-65229373498e/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -18406,11 +18406,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sys
-Version: v0.14.0
+Version: v0.15.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.14.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.15.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
@@ -18443,11 +18443,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/text
-Version: v0.13.0
+Version: v0.14.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.14.0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 

changelogs/8.11.asciidoc

@@ -3,10 +3,19 @@
 
 https://github.com/elastic/apm-server/compare/8.10\...8.11[View commits]
 
+* <<release-notes-8.11.3>>
 * <<release-notes-8.11.2>>
 * <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
 
+[float]
+[[release-notes-8.11.3]]
+=== APM version 8.11.3
+
+https://github.com/elastic/apm-server/compare/v8.11.2\...v8.11.3[View commits]
+
+No significant changes.
+
 [float]
 [[release-notes-8.11.2]]
 === APM version 8.11.2

changelogs/8.12.asciidoc

@@ -0,0 +1,30 @@
+[[release-notes-8.12]]
+== APM version 8.12
+
+https://github.com/elastic/apm-server/compare/8.11\...8.12[View commits]
+
+* <<release-notes-8.12.0>>
+
+[float]
+[[release-notes-8.12.0]]
+=== APM version 8.12.0
+
+[float]
+==== Breaking Changes
+- The unsupported apm_data_stream_migration ingest pipeline has been removed {pull}12102[12102].
+- "publish_ready" is always false in the "GET /" response until events are received by apm-server {pull}12150[12150]
+
+[float]
+==== Bug fixes
+- Switch built-in ILM rollover policies from max_size to max_primary_shard_size {pull}11729[11729].
+- Handle empty request bodies gracefully {pull}12242[12242].
+- Wrap processor errors to provide more details {pull}12242[12242].
+- Require only one of exception.{type,message} for log exceptions {pull}12242[12242].
+
+[float]
+==== Added
+- Update container base image to Ubuntu 22.04 {pull}11778[11778]
+- Add support for returning partial success response in OTLP input {pull}11883[11883]
+- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
+- Field mappings have been added for various formerly unindexed fields {pull}12102[12102]
+- We now assert that index templates are installed by attempting to create data streams {pull}12150[12150]