From d636ad3eb5b44cb0f5989eea2b7e44cbae87958a Mon Sep 17 00:00:00 2001
From: apmmachine <58790750+apmmachine@users.noreply.github.com>
Date: Fri, 15 Dec 2023 08:15:44 -0500
Subject: [PATCH 01/14] [updatecli] Bump elastic stack version to
 8.13.0-6av99u5d (#12169)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* test: update approvals doc

---------

Co-authored-by: apmmachine <apmmachine@users.noreply.github.com>
Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
---
 docker-compose.yml                                          | 6 +++---
 systemtest/approvals/TestIntake/Errors.approved.json        | 6 ------
 systemtest/approvals/TestIntake/Events.approved.json        | 6 ------
 systemtest/approvals/TestIntake/Transactions.approved.json  | 6 ------
 .../TestIntake/TransactionsHugeTraces.approved.json         | 6 ------
 testing/infra/k8s/base/stack/apm-server.yaml                | 2 +-
 testing/infra/k8s/base/stack/elasticsearch.yaml             | 2 +-
 testing/infra/k8s/base/stack/kibana.yaml                    | 2 +-
 8 files changed, 6 insertions(+), 30 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index db60d601737..24dd21884ed 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-33e8d7e1-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-6av99u5d-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.12.0-33e8d7e1-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.13.0-6av99u5d-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.12.0-33e8d7e1-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.13.0-6av99u5d-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"
diff --git a/systemtest/approvals/TestIntake/Errors.approved.json b/systemtest/approvals/TestIntake/Errors.approved.json
index 95efbeba3f6..466be37c138 100644
--- a/systemtest/approvals/TestIntake/Errors.approved.json
+++ b/systemtest/approvals/TestIntake/Errors.approved.json
@@ -27,12 +27,6 @@
         "client.geo.location": [
             "dynamic"
         ],
-        "client.geo.region_iso_code": [
-            "dynamic"
-        ],
-        "client.geo.region_name": [
-            "dynamic"
-        ],
         "client.ip": [
             "12.53.12.1"
         ],
diff --git a/systemtest/approvals/TestIntake/Events.approved.json b/systemtest/approvals/TestIntake/Events.approved.json
index 01730dcfb7b..e03a5c1f8ea 100644
--- a/systemtest/approvals/TestIntake/Events.approved.json
+++ b/systemtest/approvals/TestIntake/Events.approved.json
@@ -791,12 +791,6 @@
         "client.geo.location": [
             "dynamic"
         ],
-        "client.geo.region_iso_code": [
-            "dynamic"
-        ],
-        "client.geo.region_name": [
-            "dynamic"
-        ],
         "client.ip": [
             "12.53.12.1"
         ],
diff --git a/systemtest/approvals/TestIntake/Transactions.approved.json b/systemtest/approvals/TestIntake/Transactions.approved.json
index 83281a6eb7b..a3df939ae5d 100644
--- a/systemtest/approvals/TestIntake/Transactions.approved.json
+++ b/systemtest/approvals/TestIntake/Transactions.approved.json
@@ -445,12 +445,6 @@
         "client.geo.location": [
             "dynamic"
         ],
-        "client.geo.region_iso_code": [
-            "dynamic"
-        ],
-        "client.geo.region_name": [
-            "dynamic"
-        ],
         "client.ip": [
             "12.53.12.1"
         ],
diff --git a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json
index 6507c06df7b..c3b48460971 100644
--- a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json
+++ b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json
@@ -24,12 +24,6 @@
         "client.geo.location": [
             "dynamic"
         ],
-        "client.geo.region_iso_code": [
-            "dynamic"
-        ],
-        "client.geo.region_name": [
-            "dynamic"
-        ],
         "client.ip": [
             "12.53.12.1"
         ],
diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml
index 7815c07e525..449c18c0114 100644
--- a/testing/infra/k8s/base/stack/apm-server.yaml
+++ b/testing/infra/k8s/base/stack/apm-server.yaml
@@ -3,7 +3,7 @@ kind: ApmServer
 metadata:
   name: apm-server
 spec:
-  version: 8.12.0-33e8d7e1-SNAPSHOT
+  version: 8.13.0-6av99u5d-SNAPSHOT
   count: 1
   http:
     tls:
diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml
index 88ae897326f..20e902ab72b 100644
--- a/testing/infra/k8s/base/stack/elasticsearch.yaml
+++ b/testing/infra/k8s/base/stack/elasticsearch.yaml
@@ -3,7 +3,7 @@ kind: Elasticsearch
 metadata:
   name: elasticsearch
 spec:
-  version: 8.12.0-33e8d7e1-SNAPSHOT
+  version: 8.13.0-6av99u5d-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin
diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml
index 233a7cd492a..c28af055490 100644
--- a/testing/infra/k8s/base/stack/kibana.yaml
+++ b/testing/infra/k8s/base/stack/kibana.yaml
@@ -3,7 +3,7 @@ kind: Kibana
 metadata:
   name: kibana
 spec:
-  version: 8.12.0-33e8d7e1-SNAPSHOT
+  version: 8.13.0-6av99u5d-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch

From 8f1008427981a7bca6ad420e3edce964c170242d Mon Sep 17 00:00:00 2001
From: kruskall <99559985+kruskall@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:28:56 +0100
Subject: [PATCH 02/14] fix: always try to remap both span and error
 stacktraces (#12221)

When an event contains both a span and an error, the error is ignored by the sourcemap
processor leading to stacktraces not being remapped.
---
 internal/sourcemap/processor.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/sourcemap/processor.go b/internal/sourcemap/processor.go
index 6b2174da390..003ce3f30d1 100644
--- a/internal/sourcemap/processor.go
+++ b/internal/sourcemap/processor.go
@@ -57,10 +57,10 @@ func (p BatchProcessor) ProcessBatch(ctx context.Context, batch *modelpb.Batch)
 		if event.GetService().GetName() == "" || event.GetService().GetVersion() == "" {
 			continue
 		}
-		switch {
-		case event.Span != nil:
+		if event.Span != nil {
 			p.processStacktraceFrames(ctx, event.Service, event.Span.Stacktrace...)
-		case event.Error != nil:
+		}
+		if event.Error != nil {
 			if event.Error.Log != nil {
 				p.processStacktraceFrames(ctx, event.Service, event.Error.Log.Stacktrace...)
 			}

From 040c836def28f368ef6f26e075951a326f6ff260 Mon Sep 17 00:00:00 2001
From: apmmachine <58790750+apmmachine@users.noreply.github.com>
Date: Fri, 15 Dec 2023 11:38:49 -0500
Subject: [PATCH 03/14] [updatecli] Bump Golang version to 1.21.5 (#12191)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: Update Dockerfile with Golang version 1.21.5

Made with ❤️️ by updatecli

* chore: Update version.asciidoc with Golang version 1.21.5

Made with ❤️️ by updatecli

* chore: Update .go-version with Golang version 1.21.5

Made with ❤️️ by updatecli

---------

Co-authored-by: apmmachine <apmmachine@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com>
---
 .go-version                 | 2 +-
 docs/version.asciidoc       | 2 +-
 packaging/docker/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.go-version b/.go-version
index 20a1265cf39..ce2dd53570b 100644
--- a/.go-version
+++ b/.go-version
@@ -1 +1 @@
-1.21.4
+1.21.5
diff --git a/docs/version.asciidoc b/docs/version.asciidoc
index 5f7517f251e..4a0cf36c40c 100644
--- a/docs/version.asciidoc
+++ b/docs/version.asciidoc
@@ -1,6 +1,6 @@
 // doc-branch can be: master, 8.0, 8.1, etc.
 :doc-branch: master
-:go-version: 1.21.4
+:go-version: 1.21.5
 :python: 3.7
 :docker: 1.12
 :docker-compose: 1.11
diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index b9f9a09c165..53beb1661ca 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -2,7 +2,7 @@ ARG BASE_IMAGE=ubuntu:22.04
 
 # Build the apm-server binary. The golang image version is kept
 # up to date with go.mod by Makefile.
-FROM golang:1.21.4 as builder
+FROM golang:1.21.5 as builder
 WORKDIR /src
 COPY go.mod go.sum /src/
 COPY internal/glog/go.mod /src/internal/glog/go.mod

From 5158fea99b6dcd0a739fe0745e40df063fc29c2f Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Sun, 17 Dec 2023 17:33:58 +0100
Subject: [PATCH 04/14] smoke-test: semver sort latest versions (#12229)

Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com>
---
 testing/smoke/lib.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/testing/smoke/lib.sh b/testing/smoke/lib.sh
index 0fe19a2d731..e1c896d4f9e 100644
--- a/testing/smoke/lib.sh
+++ b/testing/smoke/lib.sh
@@ -17,7 +17,8 @@ get_versions() {
     local RC=0
     RES=$(curl_fail -H "Authorization: ApiKey ${EC_API_KEY}" ${EC_VERSION_ENDPOINT}) || RC=$?
     if [ $RC -ne 0 ]; then echo "${RES}"; fi
-    VERSIONS=$(echo "${RES}" | jq -r -c '[.stacks[].version | select(. | contains("-") | not)] | sort')
+    # NOTE: jq with semver requires some numeric transformation with sort_by
+    VERSIONS=$(echo "${RES}" | jq -r -c '[.stacks[].version | select(. | contains("-") | not)] | sort_by(.| split(".") | map(tonumber))')
 }
 
 get_latest_patch() {
@@ -38,7 +39,9 @@ get_latest_snapshot() {
     local RC=0
     RES=$(curl_fail -H "Authorization: ApiKey ${EC_API_KEY}" ${EC_VERSION_ENDPOINT}) || RC=$?
     if [ $RC -ne 0 ]; then echo "${RES}"; fi
-    VERSIONS=$(echo "${RES}" | jq -r -c '[.stacks[].version | select(. | contains("-"))] | sort')
+    # NOTE: semver with SNAPSHOT is not working when using the sort_by function in jq,
+    #       that's the reason for transforming the SNAPSHOT in a semver 4 digits.
+    VERSIONS=$(echo "${RES}" | jq -r -c '[.stacks[].version | select(. | contains("-SNAPSHOT"))] | sort' | sed 's#-SNAPSHOT#.0#g' | jq -r -c ' sort_by(.| split(".") | map(tonumber))' | sed 's#.0"#-SNAPSHOT"#g' | jq -r -c .)
 }
 
 terraform_init() {

From b2acab6208b2120b6913a0641b9859270e3c0a5e Mon Sep 17 00:00:00 2001
From: apmmachine <58790750+apmmachine@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:50:26 -0500
Subject: [PATCH 05/14] [updatecli] Bump elastic stack version to
 8.13.0-h30gube9 (#12235)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

---------

Co-authored-by: apmmachine <apmmachine@users.noreply.github.com>
---
 docker-compose.yml                              | 6 +++---
 testing/infra/k8s/base/stack/apm-server.yaml    | 2 +-
 testing/infra/k8s/base/stack/elasticsearch.yaml | 2 +-
 testing/infra/k8s/base/stack/kibana.yaml        | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 24dd21884ed..6065d07e2fc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-6av99u5d-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-h30gube9-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.13.0-6av99u5d-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.13.0-h30gube9-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.13.0-6av99u5d-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.13.0-h30gube9-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"
diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml
index 449c18c0114..49dbb026ebe 100644
--- a/testing/infra/k8s/base/stack/apm-server.yaml
+++ b/testing/infra/k8s/base/stack/apm-server.yaml
@@ -3,7 +3,7 @@ kind: ApmServer
 metadata:
   name: apm-server
 spec:
-  version: 8.13.0-6av99u5d-SNAPSHOT
+  version: 8.13.0-h30gube9-SNAPSHOT
   count: 1
   http:
     tls:
diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml
index 20e902ab72b..2457802ff45 100644
--- a/testing/infra/k8s/base/stack/elasticsearch.yaml
+++ b/testing/infra/k8s/base/stack/elasticsearch.yaml
@@ -3,7 +3,7 @@ kind: Elasticsearch
 metadata:
   name: elasticsearch
 spec:
-  version: 8.13.0-6av99u5d-SNAPSHOT
+  version: 8.13.0-h30gube9-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin
diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml
index c28af055490..c201e698b79 100644
--- a/testing/infra/k8s/base/stack/kibana.yaml
+++ b/testing/infra/k8s/base/stack/kibana.yaml
@@ -3,7 +3,7 @@ kind: Kibana
 metadata:
   name: kibana
 spec:
-  version: 8.13.0-6av99u5d-SNAPSHOT
+  version: 8.13.0-h30gube9-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch

From e74146f771aaf0d35371c003af437e34e9b0c5c5 Mon Sep 17 00:00:00 2001
From: apmmachine <58790750+apmmachine@users.noreply.github.com>
Date: Mon, 18 Dec 2023 14:16:31 -0500
Subject: [PATCH 06/14] [updatecli] Update to elastic/beats@a633696068ed
 (#12164)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: Update to elastic/beats@26bbb77d4bb7

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@16b713b6f919

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@a8d1567d9286

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@89e4737d1be5

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@cc8f5a109744

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@06a8c09655b0

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@ef1bce5cfc69

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@d11eac9452a2

Made with ❤️️ by updatecli

* chore: Update to elastic/beats@a633696068ed

Made with ❤️️ by updatecli

---------

Co-authored-by: apmmachine <apmmachine@users.noreply.github.com>
Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
---
 NOTICE.txt | 12 ++++++------
 go.mod     |  6 +++---
 go.sum     | 16 ++++++++--------
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/NOTICE.txt b/NOTICE.txt
index e2d20764e6b..08b046d12e2 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20231204124921-be42a3369f94
+Version: v7.0.0-alpha2.0.20231218141841-a633696068ed
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231204124921-be42a3369f94/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231218141841-a633696068ed/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.9.0
+Version: v0.9.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.1/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -1884,11 +1884,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.11.1
+Version: v1.11.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.11.1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.11.2/LICENSE.txt:
 
 
                                  Apache License
diff --git a/go.mod b/go.mod
index 10146be6e82..984bda2faec 100644
--- a/go.mod
+++ b/go.mod
@@ -10,14 +10,14 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
 	github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231204124921-be42a3369f94
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed
 	github.com/elastic/elastic-agent-client/v7 v7.5.0
 	github.com/elastic/elastic-agent-libs v0.7.2
-	github.com/elastic/elastic-agent-system-metrics v0.9.0
+	github.com/elastic/elastic-agent-system-metrics v0.9.1
 	github.com/elastic/gmux v0.3.1
 	github.com/elastic/go-docappender v0.2.1-0.20231207080152-44217001fa53
 	github.com/elastic/go-elasticsearch/v8 v8.11.1
-	github.com/elastic/go-sysinfo v1.11.1
+	github.com/elastic/go-sysinfo v1.11.2
 	github.com/elastic/go-ucfg v0.8.6
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible
 	github.com/gofrs/flock v0.8.1
diff --git a/go.sum b/go.sum
index a60e7a158f9..7fbf63ddec4 100644
--- a/go.sum
+++ b/go.sum
@@ -134,18 +134,18 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
 github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5 h1:URlDi0TZetSAo8KAIbaMZQhM5EG8+SDgs0jTen7WvMg=
 github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231204124921-be42a3369f94 h1:HGuZZvCTJHfw67Kv/3Dko1HRx2oWNc3ktz9xL7xRyu8=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231204124921-be42a3369f94/go.mod h1:C3d6os2PiO+aRDft33aQq94OwX0w83tut1R19QSWGB4=
-github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA=
-github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed h1:2ABWyeeCuBbi4ReJFeEAvfPUctfYLc0x5x3ZaQCAuWs=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed/go.mod h1:IyDGwc1d+ERmSiMOZZ2mJIJfCnFnIl85zQP7ZvtkSfY=
+github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U=
+github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
 github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA=
 github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0=
 github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o=
 github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.9.0 h1:sA3pSk+awH4g45M8tjcwd9U6uVQvNzUpKS8ajS3DuUk=
-github.com/elastic/elastic-agent-system-metrics v0.9.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
+github.com/elastic/elastic-agent-system-metrics v0.9.1 h1:r0ofKHgPpl+W09ie7tzGcCDC0d4NZbQUv37rSgHf4FM=
+github.com/elastic/elastic-agent-system-metrics v0.9.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
 github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/gmux v0.3.1 h1:ghT+5Y9jW5udQTBrhDeMOtQVdRBwk9Y2h1o+4Rb7mhE=
@@ -158,8 +158,8 @@ github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt
 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko=
 github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
 github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
-github.com/elastic/go-sysinfo v1.11.1 h1:g9mwl05njS4r69TisC+vwHWTSKywZFYYUu3so3T/Lao=
-github.com/elastic/go-sysinfo v1.11.1/go.mod h1:6KQb31j0QeWBDF88jIdWSxE8cwoOB9tO4Y4osN7Q70E=
+github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4=
+github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ=
 github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0=
 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=

From 9d8e17ccc14791c46e3e66c531d5e51ea02f19f6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 18 Dec 2023 15:18:38 -0500
Subject: [PATCH 07/14] build(deps): bump the github-actions group with 1
 update (#12232)

Bumps the github-actions group with 1 update: [actions/upload-artifact](https://github.com/actions/upload-artifact).

- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/benchmarks.yml | 4 ++--
 .github/workflows/ci.yml         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 1ffc720dffe..e86090601d9 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -134,7 +134,7 @@ jobs:
           $PNG_REPORT_FILE
 
       - name: Upload PNG
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: kibana-png-report
           path: ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }}
@@ -150,7 +150,7 @@ jobs:
           echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT"
 
       - name: Upload benchmark result
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: benchmark-result
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6952bde8d0c..6702ff56074 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -71,7 +71,7 @@ jobs:
           go run -modfile=tools/go.mod gotest.tools/gotestsum \
           --no-color -f standard-quiet --jsonfile "$OUTPUT_JSON_FILE" --junitfile "$OUTPUT_JUNIT_FILE" \
           --raw-command -- make system-test
-      - uses: actions/upload-artifact@v3  # upload test results
+      - uses: actions/upload-artifact@v4  # upload test results
         if: ${{ always() }} # run this step even if previous step failed
         with:
           name: test-results

From b5547a5864e0171d1f86da987cada33dc99ef8a9 Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Tue, 19 Dec 2023 13:14:35 +0100
Subject: [PATCH 08/14] ci(release): automate release steps (#12234)

---
 .github/workflows/opentelemetry.yml     |  6 +++++-
 .github/workflows/run-minor-release.yml | 22 ++++++++++++++++++++++
 .github/workflows/run-patch-release.yml | 22 ++++++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/run-minor-release.yml
 create mode 100644 .github/workflows/run-patch-release.yml

diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml
index d7f8ffca444..2274ed913eb 100644
--- a/.github/workflows/opentelemetry.yml
+++ b/.github/workflows/opentelemetry.yml
@@ -3,13 +3,17 @@ name: OpenTelemetry Export Trace
 on:
   workflow_run:
     workflows:
+      - benchmarks
       - bump-elastic-stack-snapshot
       - bump-golang
       - ci
       - microbenchmark
-      - system-test-reporter
+      - run-minor-release
+      - run-patch-release
+      - smoke-tests
       - smoke-tests-os
       - smoke-tests-ess
+      - system-test-reporter
       - Terraform Format
       - update-beats
     types: [completed]
diff --git a/.github/workflows/run-minor-release.yml b/.github/workflows/run-minor-release.yml
new file mode 100644
index 00000000000..c00184ae8e6
--- /dev/null
+++ b/.github/workflows/run-minor-release.yml
@@ -0,0 +1,22 @@
+---
+name: run-minor-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version (semver format: major.minor.patch)'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  SLACK_CHANNEL: "#apm-server-test-release"
+  
+jobs:
+  run-minor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
diff --git a/.github/workflows/run-patch-release.yml b/.github/workflows/run-patch-release.yml
new file mode 100644
index 00000000000..3f0cb3aa3d7
--- /dev/null
+++ b/.github/workflows/run-patch-release.yml
@@ -0,0 +1,22 @@
+---
+name: run-patch-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version (semver format: major.minor.patch)'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  SLACK_CHANNEL: "#apm-server-test-release"
+
+jobs:
+  run-patch:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4

From 54e367c5baa6fb42463d6507f433e2327e2008e3 Mon Sep 17 00:00:00 2001
From: Andrew Wilkins <axw@elastic.co>
Date: Tue, 19 Dec 2023 23:34:54 +0800
Subject: [PATCH 09/14] Revert "Revise wait_for_integration ES implementation
 (#12150)" (#12240)

This reverts commit 3e54384bf79d53c776a0a3d2f0480daa4f432521.
---
 internal/beater/beater.go            | 46 +++++---------
 internal/beater/beatertest/server.go |  3 +-
 internal/beater/checkintegration.go  | 91 +++++++++++++++++-----------
 internal/beater/server_test.go       | 46 +++++---------
 internal/beater/waitready.go         | 23 ++++---
 5 files changed, 99 insertions(+), 110 deletions(-)

diff --git a/internal/beater/beater.go b/internal/beater/beater.go
index 330753510dd..aee736e3fa1 100644
--- a/internal/beater/beater.go
+++ b/internal/beater/beater.go
@@ -25,7 +25,6 @@ import (
 	"net/http"
 	"os"
 	"runtime"
-	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -43,9 +42,11 @@ import (
 	_ "google.golang.org/grpc/encoding/gzip"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
 	"github.com/elastic/beats/v7/libbeat/instrumentation"
 	"github.com/elastic/beats/v7/libbeat/licenser"
 	"github.com/elastic/beats/v7/libbeat/outputs"
+	esoutput "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch"
 	"github.com/elastic/beats/v7/libbeat/publisher/pipeline"
 	"github.com/elastic/beats/v7/libbeat/publisher/pipetool"
 	agentconfig "github.com/elastic/elastic-agent-libs/config"
@@ -346,13 +347,7 @@ func (s *Runner) Run(ctx context.Context) error {
 	// any events to Elasticsearch before the integration is ready.
 	publishReady := make(chan struct{})
 	drain := make(chan struct{})
-	startWaitReady := make(chan struct{})
-	var waitReadyOnce sync.Once
 	g.Go(func() error {
-		select {
-		case <-ctx.Done():
-		case <-startWaitReady:
-		}
 		if err := s.waitReady(ctx, kibanaClient, tracer); err != nil {
 			// One or more preconditions failed; drop events.
 			close(drain)
@@ -363,25 +358,24 @@ func (s *Runner) Run(ctx context.Context) error {
 		close(publishReady)
 		return nil
 	})
-	prePublish := func(ctx context.Context) error {
-		waitReadyOnce.Do(func() {
-			close(startWaitReady)
-		})
+	callbackUUID, err := esoutput.RegisterConnectCallback(func(*eslegclient.Connection) error {
 		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-drain:
-			return errServerShuttingDown
 		case <-publishReady:
+			return nil
+		default:
 		}
-		return nil
+		return errors.New("not ready for publishing events")
+	})
+	if err != nil {
+		return err
 	}
+	defer esoutput.DeregisterConnectCallback(callbackUUID)
 	newElasticsearchClient := func(cfg *elasticsearch.Config) (*elasticsearch.Client, error) {
 		httpTransport, err := elasticsearch.NewHTTPTransport(cfg)
 		if err != nil {
 			return nil, err
 		}
-		transport := &waitReadyRoundTripper{Transport: httpTransport, onBulk: prePublish}
+		transport := &waitReadyRoundTripper{Transport: httpTransport, ready: publishReady, drain: drain}
 		return elasticsearch.NewClientParams(elasticsearch.ClientParams{
 			Config:    cfg,
 			Transport: transport,
@@ -438,7 +432,7 @@ func (s *Runner) Run(ctx context.Context) error {
 	// Create the BatchProcessor chain that is used to process all events,
 	// including the metrics aggregated by APM Server.
 	finalBatchProcessor, closeFinalBatchProcessor, err := s.newFinalBatchProcessor(
-		tracer, newElasticsearchClient, memLimitGB, prePublish,
+		tracer, newElasticsearchClient, memLimitGB,
 	)
 	if err != nil {
 		return err
@@ -653,9 +647,7 @@ func (s *Runner) waitReady(
 			return errors.New("cannot wait for integration without either Kibana or Elasticsearch config")
 		}
 		preconditions = append(preconditions, func(ctx context.Context) error {
-			return checkIndexTemplatesInstalled(
-				ctx, kibanaClient, esOutputClient, s.config.DataStreams.Namespace, s.logger,
-			)
+			return checkIntegrationInstalled(ctx, kibanaClient, esOutputClient, s.logger)
 		})
 	}
 
@@ -680,13 +672,12 @@ func (s *Runner) newFinalBatchProcessor(
 	tracer *apm.Tracer,
 	newElasticsearchClient func(cfg *elasticsearch.Config) (*elasticsearch.Client, error),
 	memLimit float64,
-	prePublish func(context.Context) error,
 ) (modelpb.BatchProcessor, func(context.Context) error, error) {
 
 	monitoring.Default.Remove("libbeat")
 	libbeatMonitoringRegistry := monitoring.Default.NewRegistry("libbeat")
 	if s.elasticsearchOutputConfig == nil {
-		return s.newLibbeatFinalBatchProcessor(tracer, prePublish, libbeatMonitoringRegistry)
+		return s.newLibbeatFinalBatchProcessor(tracer, libbeatMonitoringRegistry)
 	}
 
 	stateRegistry := monitoring.GetNamespace("state").GetRegistry()
@@ -838,7 +829,6 @@ func docappenderConfig(
 
 func (s *Runner) newLibbeatFinalBatchProcessor(
 	tracer *apm.Tracer,
-	prePublish func(context.Context) error,
 	libbeatMonitoringRegistry *monitoring.Registry,
 ) (modelpb.BatchProcessor, func(context.Context) error, error) {
 	// When the publisher stops cleanly it will close its pipeline client,
@@ -899,13 +889,7 @@ func (s *Runner) newLibbeatFinalBatchProcessor(
 		}
 		return acker.Wait(ctx)
 	}
-	processor := modelprocessor.Chained{
-		modelpb.ProcessBatchFunc(func(ctx context.Context, batch *modelpb.Batch) error {
-			return prePublish(ctx)
-		}),
-		publisher,
-	}
-	return processor, stop, nil
+	return publisher, stop, nil
 }
 
 const sourcemapIndex = ".apm-source-map"
diff --git a/internal/beater/beatertest/server.go b/internal/beater/beatertest/server.go
index 63109ac25ae..9812e3cb6e7 100644
--- a/internal/beater/beatertest/server.go
+++ b/internal/beater/beatertest/server.go
@@ -108,8 +108,7 @@ func NewUnstartedServer(t testing.TB, opts ...option) *Server {
 	require.NoError(t, err)
 	if !outputConfig.Output.IsSet() {
 		err = cfg.Merge(map[string]any{
-			"output.null":     map[string]any{},
-			"queue.mem.flush": map[string]any{"min_events": 1, "timeout": "1ns"},
+			"output.null": map[string]any{},
 		})
 		require.NoError(t, err)
 	}
diff --git a/internal/beater/checkintegration.go b/internal/beater/checkintegration.go
index 1db8bf3e84c..f0fe62ebc59 100644
--- a/internal/beater/checkintegration.go
+++ b/internal/beater/checkintegration.go
@@ -24,26 +24,23 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
 
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/elastic/elastic-agent-libs/logp"
 
 	"github.com/elastic/apm-server/internal/elasticsearch"
 	"github.com/elastic/apm-server/internal/kibana"
-	"github.com/elastic/go-elasticsearch/v8/typedapi/indices/createdatastream"
-	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/elastic/go-elasticsearch/v8/esapi"
 )
 
-// checkIndexTemplatesInstalled checks if the APM index templates are installed by querying the
-// APM integration status via Kibana, or by attempting to create a data stream via Elasticsearch,
-// returning nil if and only if it is installed.
-func checkIndexTemplatesInstalled(
+// checkIntegrationInstalled checks if the APM integration is installed by querying Kibana
+// and/or Elasticsearch, returning nil if and only if it is installed.
+func checkIntegrationInstalled(
 	ctx context.Context,
 	kibanaClient *kibana.Client,
 	esClient *elasticsearch.Client,
-	namespace string,
 	logger *logp.Logger,
 ) (err error) {
 	defer func() {
@@ -56,36 +53,37 @@ func checkIndexTemplatesInstalled(
 			}
 		}
 	}()
-	if esClient != nil {
-		installed, err := checkCreateDataStream(ctx, esClient, namespace)
-		if err != nil {
-			return fmt.Errorf("error checking Elasticsearch index template setup: %w", err)
-		}
-		if !installed {
-			return errors.New("index templates not installed")
-		}
-		return nil
-	}
 	if kibanaClient != nil {
-		installed, err := checkIntegrationInstalled(ctx, kibanaClient, logger)
+		installed, err := checkIntegrationInstalledKibana(ctx, kibanaClient, logger)
 		if err != nil {
 			// We only return the Kibana error if we have no Elasticsearch client,
 			// as we may not have sufficient privileges to query the Fleet API.
 			if esClient == nil {
 				return fmt.Errorf("error querying Kibana for integration package status: %w", err)
 			}
-		}
-		if !installed {
+		} else if !installed {
 			// We were able to query Kibana, but the package is not yet installed.
+			// We should continue querying the package status via Kibana, as it is
+			// more authoritative than checking for index template installation.
 			return errors.New("integration package not yet installed")
 		}
+		// Fall through and query Elasticsearch (if we have a client). Kibana may prematurely
+		// report packages as installed: https://github.com/elastic/kibana/issues/108649
+	}
+	if esClient != nil {
+		installed, err := checkIntegrationInstalledElasticsearch(ctx, esClient, logger)
+		if err != nil {
+			return fmt.Errorf("error querying Elasticsearch for integration index templates: %w", err)
+		} else if !installed {
+			return errors.New("integration index templates not installed")
+		}
 	}
 	return nil
 }
 
 // checkIntegrationInstalledKibana checks if the APM integration package
 // is installed by querying Kibana.
-func checkIntegrationInstalled(ctx context.Context, kibanaClient *kibana.Client, logger *logp.Logger) (bool, error) {
+func checkIntegrationInstalledKibana(ctx context.Context, kibanaClient *kibana.Client, logger *logp.Logger) (bool, error) {
 	resp, err := kibanaClient.Send(ctx, "GET", "/api/fleet/epm/packages/apm", nil, nil, nil)
 	if err != nil {
 		return false, err
@@ -108,22 +106,41 @@ func checkIntegrationInstalled(ctx context.Context, kibanaClient *kibana.Client,
 	return result.Response.Status == "installed", nil
 }
 
-// checkCreateDataStream attempts to create a traces-apm-<namespace> data stream,
-// returning an error if it could not be created. This will fail if there is no
-// index template matching the pattern.
-func checkCreateDataStream(ctx context.Context, esClient *elasticsearch.Client, namespace string) (bool, error) {
-	if _, err := createdatastream.NewCreateDataStreamFunc(esClient)("traces-apm-" + namespace).Do(ctx); err != nil {
-		var esError *types.ElasticsearchError
-		if errors.As(err, &esError) {
-			cause := esError.ErrorCause
-			if cause.Type == "resource_already_exists_exception" {
-				return true, nil
+func checkIntegrationInstalledElasticsearch(ctx context.Context, esClient *elasticsearch.Client, _ *logp.Logger) (bool, error) {
+	// TODO(axw) generate the list of expected index templates.
+	templates := []string{
+		"traces-apm",
+		"traces-apm.sampled",
+		"metrics-apm.app",
+		"metrics-apm.internal",
+		"logs-apm.error",
+	}
+	for _, intervals := range []string{"1m", "10m", "60m"} {
+		for _, ds := range []string{"metrics-apm.transaction", "metrics-apm.service_transaction", "metrics-apm.service_destination", "metrics-apm.service_summary"} {
+			templates = append(templates, fmt.Sprintf("%s.%s", ds, intervals))
+		}
+	}
+	// IndicesGetIndexTemplateRequest accepts a slice of template names,
+	// but the REST API expects just one index template name. Query them
+	// in parallel.
+	g, ctx := errgroup.WithContext(ctx)
+	for _, template := range templates {
+		template := template // copy for closure
+		g.Go(func() error {
+			req := esapi.IndicesGetIndexTemplateRequest{Name: template}
+			resp, err := req.Do(ctx, esClient)
+			if err != nil {
+				return err
 			}
-			if cause.Reason != nil && strings.HasPrefix(*cause.Reason, "no matching index template") {
-				return false, nil
+			defer resp.Body.Close()
+
+			if resp.IsError() {
+				body, _ := io.ReadAll(resp.Body)
+				return fmt.Errorf("unexpected HTTP status: %s (%s)", resp.Status(), bytes.TrimSpace(body))
 			}
-		}
-		return false, err
+			return nil
+		})
 	}
-	return true, nil
+	err := g.Wait()
+	return err == nil, err
 }
diff --git a/internal/beater/server_test.go b/internal/beater/server_test.go
index 98429cd9247..1249a41c4af 100644
--- a/internal/beater/server_test.go
+++ b/internal/beater/server_test.go
@@ -332,7 +332,7 @@ func TestServerOTLPGRPC(t *testing.T) {
 
 func TestServerWaitForIntegrationKibana(t *testing.T) {
 	var requests int64
-	requestCh := make(chan struct{}, 3)
+	requestCh := make(chan struct{})
 	mux := http.NewServeMux()
 	mux.HandleFunc("/api/status", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(`{"version":{"number":"1.2.3"}}`))
@@ -363,14 +363,6 @@ func TestServerWaitForIntegrationKibana(t *testing.T) {
 		},
 	})))
 
-	// Send some events to the server. They should be accepted and enqueued.
-	req := makeTransactionRequest(t, srv.URL)
-	req.Header.Add("Content-Type", "application/x-ndjson")
-	resp, err := srv.Client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
-	resp.Body.Close()
-
 	timeout := time.After(10 * time.Second)
 	for i := 0; i < 3; i++ {
 		select {
@@ -395,8 +387,8 @@ func TestServerWaitForIntegrationKibana(t *testing.T) {
 
 func TestServerWaitForIntegrationElasticsearch(t *testing.T) {
 	var mu sync.Mutex
-	var createDataStreamRequests int
-	createDataStreamRequestsCh := make(chan int)
+	var tracesRequests int
+	tracesRequestsCh := make(chan int)
 	bulkCh := make(chan struct{}, 1)
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -405,24 +397,17 @@ func TestServerWaitForIntegrationElasticsearch(t *testing.T) {
 		// elasticsearch client to send bulk requests.
 		fmt.Fprintln(w, `{"version":{"number":"1.2.3"}}`)
 	})
-	mux.HandleFunc("/_data_stream/", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/_index_template/", func(w http.ResponseWriter, r *http.Request) {
 		mu.Lock()
 		defer mu.Unlock()
-		name := path.Base(r.URL.Path)
-		if name != "traces-apm-testing" {
-			panic("unexpected data stream name: " + name)
-		}
-		createDataStreamRequests++
-		switch createDataStreamRequests {
-		case 1:
-			w.WriteHeader(500)
-		case 2:
-			w.WriteHeader(400)
-			w.Write([]byte(`{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"no matching index template found for data stream [traces-apm-testing]"}],"type":"illegal_argument_exception","reason":"no matching index template found for data stream [traces-apm-testing]"},"status":400}`))
-		case 3:
-			w.Write([]byte(`{"acknowledged":true}`))
+		template := path.Base(r.URL.Path)
+		if template == "traces-apm" {
+			tracesRequests++
+			if tracesRequests == 1 {
+				w.WriteHeader(404)
+			}
+			tracesRequestsCh <- tracesRequests
 		}
-		createDataStreamRequestsCh <- createDataStreamRequests
 	})
 	mux.HandleFunc("/_bulk", func(w http.ResponseWriter, r *http.Request) {
 		select {
@@ -437,7 +422,6 @@ func TestServerWaitForIntegrationElasticsearch(t *testing.T) {
 		"apm-server": map[string]interface{}{
 			"wait_ready_interval":               "100ms",
 			"data_streams.wait_for_integration": true,
-			"data_streams.namespace":            "testing",
 		},
 		"output.elasticsearch": map[string]interface{}{
 			"hosts":       []string{elasticsearchServer.URL},
@@ -472,8 +456,8 @@ func TestServerWaitForIntegrationElasticsearch(t *testing.T) {
 	var done bool
 	for !done {
 		select {
-		case n := <-createDataStreamRequestsCh:
-			done = n == 3
+		case n := <-tracesRequestsCh:
+			done = n == 2
 		case <-timeout:
 			t.Fatal("timed out waiting for request")
 		}
@@ -487,7 +471,7 @@ func TestServerWaitForIntegrationElasticsearch(t *testing.T) {
 	}
 
 	logs := srv.Logs.FilterMessageSnippet("please install the apm integration")
-	assert.Len(t, logs.All(), 2, "couldn't find remediation message logs")
+	assert.Len(t, logs.All(), 1, "couldn't find remediation message logs")
 
 	// Healthcheck should now report that the server is publish-ready.
 	resp, err = srv.Client.Get(srv.URL + api.RootPath)
@@ -506,7 +490,7 @@ func TestServerFailedPreconditionDoesNotIndex(t *testing.T) {
 		// elasticsearch client to send bulk requests.
 		fmt.Fprintln(w, `{"version":{"number":"1.2.3"}}`)
 	})
-	mux.HandleFunc("/_data_stream/traces-apm-default", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/_index_template/", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(404)
 	})
 	mux.HandleFunc("/_bulk", func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/beater/waitready.go b/internal/beater/waitready.go
index 50811a4d688..7bfaa6b04fd 100644
--- a/internal/beater/waitready.go
+++ b/internal/beater/waitready.go
@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"net/http"
-	"path"
 	"time"
 
 	"go.elastic.co/apm/v2"
@@ -72,21 +71,27 @@ func waitReady(
 }
 
 // waitReadyRoundTripper wraps a *net/http.Transport, ensuring the server's
-// indexing preconditions have been satisfied prior to allowing any indexing
-// requests through. This is used to ensure we don't index any documents prior
-// to the data stream index templates being ready.
+// indexing preconditions have been satisfied by waiting for "ready" channel
+// to be signalled, prior to allowing any requests through.
+//
+// This is used to prevent elasticsearch clients from proceeding with requests
+// until the APM integration is installed to ensure we don't index any documents
+// prior to the data stream index templates being ready.
 type waitReadyRoundTripper struct {
 	*http.Transport
-	onBulk func(context.Context) error
+	ready <-chan struct{}
+	drain <-chan struct{}
 }
 
 var errServerShuttingDown = errors.New("server shutting down")
 
 func (c *waitReadyRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
-	if path.Base(r.URL.Path) == "_bulk" {
-		if err := c.onBulk(r.Context()); err != nil {
-			return nil, err
-		}
+	select {
+	case <-c.drain:
+		return nil, errServerShuttingDown
+	case <-c.ready:
+	case <-r.Context().Done():
+		return nil, r.Context().Err()
 	}
 	return c.Transport.RoundTrip(r)
 }

From 67ec9040a2da23fa223878c617e05591a2f19db2 Mon Sep 17 00:00:00 2001
From: Silvia Mitter <silvia.mitter@elastic.co>
Date: Tue, 19 Dec 2023 16:59:56 +0100
Subject: [PATCH 10/14] Update apm data (#12242)

* Update to apm-data version to handle empty rquest bodies gracefully

* Update apm-data to wrap processor errors

* Update apm-data to require one exception.{type,message} for log exceptions
---
 NOTICE.txt                                           |  4 ++--
 changelogs/8.12.asciidoc                             |  3 +++
 go.mod                                               |  2 +-
 go.sum                                               |  4 ++--
 internal/beater/api/intake/handler_test.go           |  4 ++--
 .../intake/test_approved/BodyReader.approved.json    |  8 --------
 .../api/intake/test_approved/EmptyBody.approved.json |  3 +++
 .../api/intake/test_approved/FullQueue.approved.json |  2 +-
 .../test_approved/FullQueueLegacy.approved.json      |  2 +-
 .../On.approved.json                                 |  8 --------
 ...tRUMHandler_NoAuthorizationRequired.approved.json |  8 --------
 internal/beater/api/mux_intake_rum_test.go           | 12 ++----------
 systemtest/rum_test.go                               |  2 +-
 13 files changed, 18 insertions(+), 44 deletions(-)
 delete mode 100644 internal/beater/api/intake/test_approved/BodyReader.approved.json
 create mode 100644 internal/beater/api/intake/test_approved/EmptyBody.approved.json
 delete mode 100644 internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_KillSwitchMiddleware/On.approved.json
 delete mode 100644 internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_NoAuthorizationRequired.approved.json

diff --git a/NOTICE.txt b/NOTICE.txt
index 08b046d12e2..58bcb31d7dc 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -362,11 +362,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/apm-data
-Version: v0.1.1-0.20231019120308-82201b991ae5
+Version: v0.1.1-0.20231120111210-bd8f3ecad5bb
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20231019120308-82201b991ae5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20231120111210-bd8f3ecad5bb/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
diff --git a/changelogs/8.12.asciidoc b/changelogs/8.12.asciidoc
index ea2d8013de7..94abff210f4 100644
--- a/changelogs/8.12.asciidoc
+++ b/changelogs/8.12.asciidoc
@@ -17,6 +17,9 @@ https://github.com/elastic/apm-server/compare/8.11\...8.12[View commits]
 [float]
 ==== Bug fixes
 - Switch built-in ILM rollover policies from max_size to max_primary_shard_size {pull}11729[11729].
+- Handle empty request bodies gracefully {pull}12242[12242].
+- Wrap processor errors to provide more details {pull}12242[12242].
+- Require only one of exception.{type,message} for log exceptions {pull}12242[12242].
 
 [float]
 ==== Added
diff --git a/go.mod b/go.mod
index 984bda2faec..714d1dee50c 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
-	github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5
+	github.com/elastic/apm-data v0.1.1-0.20231120111210-bd8f3ecad5bb
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed
 	github.com/elastic/elastic-agent-client/v7 v7.5.0
 	github.com/elastic/elastic-agent-libs v0.7.2
diff --git a/go.sum b/go.sum
index 7fbf63ddec4..d87ed10ad75 100644
--- a/go.sum
+++ b/go.sum
@@ -132,8 +132,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw=
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
-github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5 h1:URlDi0TZetSAo8KAIbaMZQhM5EG8+SDgs0jTen7WvMg=
-github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo=
+github.com/elastic/apm-data v0.1.1-0.20231120111210-bd8f3ecad5bb h1:mpfOadQBl6amJQzbXa0d9oy9GTvrlx0kqAb4R1soNoU=
+github.com/elastic/apm-data v0.1.1-0.20231120111210-bd8f3ecad5bb/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo=
 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed h1:2ABWyeeCuBbi4ReJFeEAvfPUctfYLc0x5x3ZaQCAuWs=
 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231218141841-a633696068ed/go.mod h1:IyDGwc1d+ERmSiMOZZ2mJIJfCnFnIl85zQP7ZvtkSfY=
 github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U=
diff --git a/internal/beater/api/intake/handler_test.go b/internal/beater/api/intake/handler_test.go
index cf15c2e7928..8cfa0c5a79d 100644
--- a/internal/beater/api/intake/handler_test.go
+++ b/internal/beater/api/intake/handler_test.go
@@ -64,14 +64,14 @@ func TestIntakeHandler(t *testing.T) {
 			}(),
 			code: http.StatusBadRequest, id: request.IDResponseErrorsValidate,
 		},
-		"BodyReader": {
+		"EmptyBody": {
 			path: "errors.ndjson",
 			r: func() *http.Request {
 				req := httptest.NewRequest(http.MethodPost, "/", nil)
 				req.Header.Set(headers.ContentType, "application/x-ndjson")
 				return req
 			}(),
-			code: http.StatusBadRequest, id: request.IDResponseErrorsValidate,
+			code: http.StatusAccepted, id: request.IDResponseValidAccepted,
 		},
 		"CompressedBodyReaderDeflateInvalid": {
 			path: "errors.ndjson",
diff --git a/internal/beater/api/intake/test_approved/BodyReader.approved.json b/internal/beater/api/intake/test_approved/BodyReader.approved.json
deleted file mode 100644
index fc636bfdcb3..00000000000
--- a/internal/beater/api/intake/test_approved/BodyReader.approved.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "accepted": 0,
-    "errors": [
-        {
-            "message": "EOF while reading metadata"
-        }
-    ]
-}
diff --git a/internal/beater/api/intake/test_approved/EmptyBody.approved.json b/internal/beater/api/intake/test_approved/EmptyBody.approved.json
new file mode 100644
index 00000000000..6b51efae3bf
--- /dev/null
+++ b/internal/beater/api/intake/test_approved/EmptyBody.approved.json
@@ -0,0 +1,3 @@
+{
+    "accepted": 0
+}
diff --git a/internal/beater/api/intake/test_approved/FullQueue.approved.json b/internal/beater/api/intake/test_approved/FullQueue.approved.json
index ce2dd7e6357..c163b675ec7 100644
--- a/internal/beater/api/intake/test_approved/FullQueue.approved.json
+++ b/internal/beater/api/intake/test_approved/FullQueue.approved.json
@@ -2,7 +2,7 @@
     "accepted": 0,
     "errors": [
         {
-            "message": "queue is full"
+            "message": "cannot handle stream: cannot process batch: queue is full"
         }
     ]
 }
diff --git a/internal/beater/api/intake/test_approved/FullQueueLegacy.approved.json b/internal/beater/api/intake/test_approved/FullQueueLegacy.approved.json
index ce2dd7e6357..c163b675ec7 100644
--- a/internal/beater/api/intake/test_approved/FullQueueLegacy.approved.json
+++ b/internal/beater/api/intake/test_approved/FullQueueLegacy.approved.json
@@ -2,7 +2,7 @@
     "accepted": 0,
     "errors": [
         {
-            "message": "queue is full"
+            "message": "cannot handle stream: cannot process batch: queue is full"
         }
     ]
 }
diff --git a/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_KillSwitchMiddleware/On.approved.json b/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_KillSwitchMiddleware/On.approved.json
deleted file mode 100644
index fc636bfdcb3..00000000000
--- a/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_KillSwitchMiddleware/On.approved.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "accepted": 0,
-    "errors": [
-        {
-            "message": "EOF while reading metadata"
-        }
-    ]
-}
diff --git a/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_NoAuthorizationRequired.approved.json b/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_NoAuthorizationRequired.approved.json
deleted file mode 100644
index fc636bfdcb3..00000000000
--- a/internal/beater/api/intake/test_approved/integration/rum/TestRUMHandler_NoAuthorizationRequired.approved.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "accepted": 0,
-    "errors": [
-        {
-            "message": "EOF while reading metadata"
-        }
-    ]
-}
diff --git a/internal/beater/api/mux_intake_rum_test.go b/internal/beater/api/mux_intake_rum_test.go
index 946a6dfc373..f50f82d915a 100644
--- a/internal/beater/api/mux_intake_rum_test.go
+++ b/internal/beater/api/mux_intake_rum_test.go
@@ -75,11 +75,7 @@ func TestRUMHandler_NoAuthorizationRequired(t *testing.T) {
 	cfg.AgentAuth.SecretToken = "1234"
 	rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath)
 	require.NoError(t, err)
-	assert.NotEqual(t, http.StatusUnauthorized, rec.Code)
-
-	expected, err := os.ReadFile(approvalPathIntakeRUM(t.Name()) + ".approved.json")
-	require.NoError(t, err)
-	assert.JSONEq(t, string(expected), rec.Body.String())
+	assert.Equal(t, http.StatusAccepted, rec.Code)
 }
 
 func TestRUMHandler_KillSwitchMiddleware(t *testing.T) {
@@ -96,11 +92,7 @@ func TestRUMHandler_KillSwitchMiddleware(t *testing.T) {
 	t.Run("On", func(t *testing.T) {
 		rec, err := requestToMuxerWithPattern(cfgEnabledRUM(), IntakeRUMPath)
 		require.NoError(t, err)
-		assert.NotEqual(t, http.StatusForbidden, rec.Code)
-
-		expected, err := os.ReadFile(approvalPathIntakeRUM(t.Name()) + ".approved.json")
-		require.NoError(t, err)
-		assert.JSONEq(t, string(expected), rec.Body.String())
+		assert.Equal(t, http.StatusAccepted, rec.Code)
 	})
 }
 
diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go
index d082d0b12b2..d48659dc205 100644
--- a/systemtest/rum_test.go
+++ b/systemtest/rum_test.go
@@ -110,7 +110,7 @@ func TestRUMAllowServiceNames(t *testing.T) {
 
 	respBody, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode, string(respBody))
-	assert.Equal(t, `{"accepted":0,"errors":[{"message":"unauthorized: anonymous access not permitted for service \"disallowed\""}]}`+"\n", string(respBody))
+	assert.Equal(t, `{"accepted":0,"errors":[{"message":"cannot handle stream: cannot process batch: unauthorized: anonymous access not permitted for service \"disallowed\""}]}`+"\n", string(respBody))
 }
 
 func TestRUMRateLimit(t *testing.T) {

From 0f695473c7c2686ac05ca95622aa46675ac7243a Mon Sep 17 00:00:00 2001
From: Colleen McGinnis <colleen.mcginnis@elastic.co>
Date: Tue, 19 Dec 2023 17:16:30 -0600
Subject: [PATCH 11/14] docs: Add tagged regions to changelog (#12251)

* add tagged regions to changelog

* add back changelogs to file paths

* rename and move breaking changes file
---
 CHANGELOG.asciidoc                       |  18 ++
 changelogs/all-breaking-changes.asciidoc | 289 +++++++++++++++++++++++
 2 files changed, 307 insertions(+)
 create mode 100644 changelogs/all-breaking-changes.asciidoc

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 2ce78ec26dd..7b32b7b3575 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -1,3 +1,20 @@
+// tag::list[]
+* <<release-notes-8.12>>
+* <<release-notes-8.11>>
+* <<release-notes-8.10>>
+* <<release-notes-8.9>>
+* <<release-notes-8.8>>
+* <<release-notes-8.7>>
+* <<release-notes-8.6>>
+* <<release-notes-8.5>>
+* <<release-notes-8.4>>
+* <<release-notes-8.3>>
+* <<release-notes-8.2>>
+* <<release-notes-8.1>>
+* <<release-notes-8.0>>
+// end::list[]
+
+// tag::includes[]
 include::./changelogs/head.asciidoc[]
 include::./changelogs/8.12.asciidoc[]
 include::./changelogs/8.11.asciidoc[]
@@ -12,3 +29,4 @@ include::./changelogs/8.3.asciidoc[]
 include::./changelogs/8.2.asciidoc[]
 include::./changelogs/8.1.asciidoc[]
 include::./changelogs/8.0.asciidoc[]
+// end::includes[]
diff --git a/changelogs/all-breaking-changes.asciidoc b/changelogs/all-breaking-changes.asciidoc
new file mode 100644
index 00000000000..58335f86b1b
--- /dev/null
+++ b/changelogs/all-breaking-changes.asciidoc
@@ -0,0 +1,289 @@
+:issue: https://github.com/elastic/apm-server/issues/
+:pull: https://github.com/elastic/apm-server/pull/
+
+[[apm-breaking]]
+=== Breaking Changes
+
+This section describes the breaking changes and deprecations introduced in this release
+and previous minor versions.
+
+// tag::811-bc[]
+[float]
+[[breaking-changes-8.11]]
+=== 8.11
+
+The following breaking changes are introduced in APM version 8.11.0:
+
+- The `ecs.version` field has been removed from APM data streams.
+This change should not impact most users as this field is not used by the APM UI.
+For more details, see https://github.com/elastic/apm-server/pull/11632[PR #11632]
+// end::811-bc[]
+
+// tag::810-bc[]
+[float]
+[[breaking-changes-8.10]]
+=== 8.10
+
+The following breaking changes are introduced in APM version 8.10.0:
+
+- Aggregated metrics now consider global labels to be part of a service's identity, and high cardinality global labels may cause services to be obscured.
+For more details, see https://github.com/elastic/apm-server/pull/11386[PR #11386].
+
+- Event protobuf encoding for tail-based sampling changed to a more efficient encoding for event timestamp and duration
+For more details, see https://github.com/elastic/apm-server/pull/11386[PR #11386].
+// end::810-bc[]
+
+// tag::87-bc[]
+[float]
+[[breaking-changes-8.7]]
+=== 8.7
+
+The following breaking changes and deprecations are introduced in APM version 8.7.0:
+
+- `transaction.failure_count` has been removed. `transaction.success_count` type has changed to `aggregated_metric_double`.
+For more details, see https://github.com/elastic/apm-server/pull/9791[PR #9791].
+
+- `transaction.success_count` has been moved to `event.success_count`.
+For more details, see https://github.com/elastic/apm-server/pull/9819[PR #9819].
+
+- Stopped indexing transaction metrics to `metrics-apm.internal`.
+For more details, see https://github.com/elastic/apm-server/pull/9846[PR #9846].
+
+- Stopped indexing span destination metrics to `metrics-apm.internal`.
+For more details, see https://github.com/elastic/apm-server/pull/9926[PR #9926].
+
+- `apmserver.aggregation.txmetrics.overflowed` metric has been renamed to `apmserver.aggregation.txmetrics.overflowed.total`.
+For more details, see https://github.com/elastic/apm-server/pull/10330[PR #10330].
+
+- Elasticsearch source mapping credentials now require access to the `.apm-source-map` index.
+For more details, see https://github.com/elastic/apm-server/pull/9722[PR #9722].
+
+- Changed APM Server default host to `127.0.0.1`.
+For more details, see https://github.com/elastic/apm-server/pull/9877[PR #9877].
+// end::87-bc[]
+
+// tag::86-bc[]
+[float]
+[[breaking-changes-8.6]]
+=== 8.6
+
+The following breaking changes and deprecations are introduced in APM version 8.6.0:
+
+[float]
+==== `apm-server.decoder.*` no longer recorded
+The stack monitoring metrics, `apm-server.decoder.*`, are no longer recorded.
+These metrics were not used by stack monitoring, so there should be no noticeable change.
+
+For more details, see https://github.com/elastic/apm-server/pull/9210[PR #9210].
+
+[float]
+==== `context.http.response.*_size` fields now enforce integer values
+New field mappings enforce integer values for `context.http.response.*_size`.
+The fields are mapped with `index: false` to minimize storage overhead.
+
+For more details, see https://github.com/elastic/apm-server/pull/9429[PR #9429].
+
+[float]
+==== `observer.id` and `observer.ephemeral_id` removed
+
+`observer.id` and `observer.ephemeral_id` are no longer added to APM documents.
+The APM UI does not currently rely on these field, so there should be no noticeable change.
+
+For more details, see https://github.com/elastic/apm-server/pull/9412[PR #9412].
+
+[float]
+==== `timeseries.instance` removed
+`timeseries.instance` has been removed from transaction metrics docs.
+The APM UI did not use this field, so there should be no noticeable change.
+
+For more details, see https://github.com/elastic/apm-server/pull/9565[PR #9565].
+
+// end::86-bc[]
+
+[float]
+[[breaking-changes-8.2]]
+=== 8.2
+
+// tag::82-bc[]
+The following breaking changes are introduced in APM version 8.2.0:
+
+[float]
+==== APM Server now emits events with `event.duration`
+
+APM Server no longer emits events with a `transaction.duration.us` or `span.duration.us`.
+Instead, events are emitted with an `event.duration`.
+An ingest pipeline sets the legacy `<event>.duration.us` field and removes the `event.duration`.
+
+This change will impact users who are not using APM Server's {es} output or the packaged ingest pipeline.
+For details, see https://github.com/elastic/apm-server/pull/7261[PR #7261].
+
+[float]
+==== Removed `observer.version_major`
+
+The field `observer.version_major` is non-standard and existed only for the APM UI to filter out legacy docs (versions <7.0).
+This check is no longer performed, so the field has been removed.
+
+For details, see https://github.com/elastic/apm-server/pull/7399[PR #7399].
+
+[float]
+==== APM Server no longer ships with System V init scripts or the go-daemon wrapper
+
+As of version 8.1.0, all Linux distributions supported by APM Server support systemd.
+As a result, APM Server no longer ships with System V init scripts or the go-daemon wrapper; use systemd instead.
+
+For details, see https://github.com/elastic/apm-server/pull/7576[PR #7576].
+
+[float]
+==== Deprecated 32-bit architectures
+
+APM Server support for 32-bit architectures has been deprecated and will be removed in a future release.
+// end::82-bc[]
+
+[float]
+[[breaking-changes-8.1]]
+=== 8.1
+
+// tag::81-bc[]
+There are no breaking changes in APM.
+// end::81-bc[]
+
+[float]
+[[breaking-changes-8.0]]
+=== 8.0
+
+// tag::80-bc[]
+The following breaking changes are introduced in APM version 8.0.
+
+[float]
+==== Indices are now manged by {fleet}
+
+All index management has been removed from APM Server;
+{fleet} is now entirely responsible for setting up index templates, index lifecycle polices,
+and index pipelines.
+
+As a part of this change, the following settings have been removed:
+
+* `apm-server.ilm.*`
+* `apm-server.register.ingest.pipeline.*`
+* `setup.*`
+
+[float]
+==== Data streams by default
+
+APM Server now only writes to well-defined data streams;
+writing to classic indices is no longer supported.
+
+As a part of this change, the following settings have been removed:
+
+* `apm-server.data_streams.enabled`
+* `output.elasticsearch.index`
+* `output.elasticsearch.indices`
+* `output.elasticsearch.pipeline`
+* `output.elasticsearch.pipelines`
+
+[float]
+==== New {es} output
+
+APM Server has a new {es} output implementation; it is no longer necessary to manually
+tune the output of APM Server.
+
+As a part of this change, the following settings have been removed:
+
+* `output.elasticsearch.bulk_max_size`
+* `output.elasticsearch.worker`
+* `queue.*`
+
+[float]
+==== New source map upload endpoint
+
+The source map upload endpoint has been removed from APM Server.
+Source maps should now be uploaded directly to {kib} instead.
+
+[float]
+==== Legacy Jaeger endpoints have been removed
+
+The legacy Jaeger gRPC and HTTP endpoints have been removed from APM Server.
+
+As a part of this change, the following settings have been removed:
+
+* `apm-server.jaeger`
+
+[float]
+==== Homebrew no longer supported
+
+APM Server no longer supports installation via Homebrew.
+
+[float]
+==== All removed and changed settings
+
+Below is a list of all **removed settings** (in alphabetical order) for
+users upgrading a standalone APM Server to {stack} version 8.0.
+
+[source,yml]
+----
+apm-server.data_streams.enabled
+apm-server.ilm.*
+apm-server.jaeger
+apm-server.register.ingest.pipeline.*
+apm-server.sampling.keep_unsampled
+output.elasticsearch.bulk_max_size
+output.elasticsearch.index
+output.elasticsearch.indices
+output.elasticsearch.pipeline
+output.elasticsearch.pipelines
+output.elasticsearch.worker
+queue.*
+setup.*
+----
+
+Below is a list of **renamed settings** (in alphabetical order) for
+users upgrading a standalone APM Server to {stack} version 8.0.
+
+[source,yml]
+----
+previous setting                --> new setting
+
+apm-server.api_key              --> apm-server.auth.api_key
+apm-server.instrumentation      --> instrumentation
+apm-server.rum.allowed_service  --> apm-server.auth.anonymous.allow_service
+apm-server.rum.event_rate       --> apm-server.auth.anonymous.rate_limit
+apm-server.secret_token         --> apm-server.auth.secret_token
+----
+
+[float]
+==== Supported {ecloud} settings
+
+Below is a list of all **supported settings** (in alphabetical order) for
+users upgrading an {ecloud} standalone cluster to {stack} version 8.0.
+Any previously supported settings not listed below will be removed when upgrading.
+
+[source,yml]
+----
+apm-server.agent.config.cache.expiration
+apm-server.aggregation.transactions.*
+apm-server.auth.anonymous.allow_agent
+apm-server.auth.anonymous.allow_service
+apm-server.auth.anonymous.rate_limit.event_limit
+apm-server.auth.anonymous.rate_limit.ip_limit
+apm-server.auth.api_key.enabled
+apm-server.auth.api_key.limit
+apm-server.capture_personal_data
+apm-server.default_service_environment
+apm-server.max_event_size
+apm-server.rum.allow_headers
+apm-server.rum.allow_origins
+apm-server.rum.enabled
+apm-server.rum.exclude_from_grouping
+apm-server.rum.library_pattern
+apm-server.rum.source_mapping.enabled
+apm-server.rum.source_mapping.cache.expiration
+logging.level
+logging.selectors
+logging.metrics.enabled
+logging.metrics.period
+max_procs
+output.elasticsearch.flush_bytes
+output.elasticsearch.flush_interval
+----
+
+// end::80-bc[]

From d01c66b90f25f7bd9c4868b4f16974bfc0e68e8e Mon Sep 17 00:00:00 2001
From: Nassim Kammah <nkammah@gmail.com>
Date: Wed, 20 Dec 2023 19:22:45 +0100
Subject: [PATCH 12/14] Remove Nassim Kammah from list of maintainers (#12269)

---
 packaging/ironbank/hardening_manifest.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packaging/ironbank/hardening_manifest.yaml b/packaging/ironbank/hardening_manifest.yaml
index d456b73c335..c30bc656bd4 100644
--- a/packaging/ironbank/hardening_manifest.yaml
+++ b/packaging/ironbank/hardening_manifest.yaml
@@ -53,9 +53,6 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - email: "nassim.kammah@elastic.co"
-    name: "Nassim Kammah"
-    username: "nassim.kammah"
   - email: "ivan.fernandez@elastic.co"
     name: "Ivan Fernandez Calvo"
     username: "ivan.fernandez"

From 5ca65a25c5a64ad5ba473aae87cd93a7b566e8bf Mon Sep 17 00:00:00 2001
From: apmmachine <58790750+apmmachine@users.noreply.github.com>
Date: Thu, 21 Dec 2023 11:14:22 -0500
Subject: [PATCH 13/14] [updatecli] Bump elastic stack version to
 8.13.0-yufkxnwm (#12246)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

* chore: Update k8s stack yaml files

Made with ❤️️ by updatecli

* chore: Update docker-compose.yml

Made with ❤️️ by updatecli

---------

Co-authored-by: apmmachine <apmmachine@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
---
 docker-compose.yml                              | 6 +++---
 testing/infra/k8s/base/stack/apm-server.yaml    | 2 +-
 testing/infra/k8s/base/stack/elasticsearch.yaml | 2 +-
 testing/infra/k8s/base/stack/kibana.yaml        | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 6065d07e2fc..6ceba73cdbc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-h30gube9-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-yufkxnwm-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.13.0-h30gube9-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.13.0-yufkxnwm-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.13.0-h30gube9-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.13.0-yufkxnwm-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"
diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml
index 49dbb026ebe..a60f8fcec4a 100644
--- a/testing/infra/k8s/base/stack/apm-server.yaml
+++ b/testing/infra/k8s/base/stack/apm-server.yaml
@@ -3,7 +3,7 @@ kind: ApmServer
 metadata:
   name: apm-server
 spec:
-  version: 8.13.0-h30gube9-SNAPSHOT
+  version: 8.13.0-yufkxnwm-SNAPSHOT
   count: 1
   http:
     tls:
diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml
index 2457802ff45..1e8466e3bef 100644
--- a/testing/infra/k8s/base/stack/elasticsearch.yaml
+++ b/testing/infra/k8s/base/stack/elasticsearch.yaml
@@ -3,7 +3,7 @@ kind: Elasticsearch
 metadata:
   name: elasticsearch
 spec:
-  version: 8.13.0-h30gube9-SNAPSHOT
+  version: 8.13.0-yufkxnwm-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin
diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml
index c201e698b79..23296617951 100644
--- a/testing/infra/k8s/base/stack/kibana.yaml
+++ b/testing/infra/k8s/base/stack/kibana.yaml
@@ -3,7 +3,7 @@ kind: Kibana
 metadata:
   name: kibana
 spec:
-  version: 8.13.0-h30gube9-SNAPSHOT
+  version: 8.13.0-yufkxnwm-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch

From 29076053edb9328049d999bd264fa60bb90577e8 Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Thu, 21 Dec 2023 20:59:22 +0100
Subject: [PATCH 14/14] buildkite: retry docker login (#12203)

Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com>
---
 .buildkite/hooks/pre-command | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index acb73845547..2b6f463b662 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -70,7 +70,7 @@ if command -v docker &>/dev/null; then
     DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field username "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_REGISTRY_SECRET=$(retry 5 vault kv get -field registry "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY_SECRET}"
+    retry 4 docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY_SECRET}"
     unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
     export DOCKER_REGISTRY_SECRET
     retry 4 docker pull --quiet docker.elastic.co/infra/release-manager:latest
@@ -80,7 +80,7 @@ if command -v docker &>/dev/null; then
     DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field username "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKERHUB_REGISTRY_SECRET=$(retry 5 vault kv get -field registry "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKERHUB_REGISTRY_SECRET}"
+    retry 4 docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKERHUB_REGISTRY_SECRET}"
     unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
 fi