Merge branch 'main' into issue-11593

NOTICE.txt

@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20231121082709-886d078ba1f0
+Version: v7.0.0-alpha2.0.20231129135154-6f32513e5037
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231121082709-886d078ba1f0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231129135154-6f32513e5037/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.8.1
+Version: v0.8.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

apmpackage/apm/changelog.yml

@@ -1,8 +1,11 @@
 - version: generated
   changes:
-    - description: Placeholder
+    - description: Add missing mappings for various fields
       type: enhancement
-      link: https://github.com/elastic/apm-server/pull/123
+      link: https://github.com/elastic/apm-server/pull/12102
+    - description: Remove unsupported 7.x data stream migration pipeline
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/12102
 - version: 8.11.0
   changes:
     - description: Add geoip processing to add client.geo.* fields for app_logs

apmpackage/apm/data_stream/error_logs/fields/fields.yml

@@ -6,6 +6,12 @@
 - name: error.culprit
   type: keyword
   description: Function call which was the primary perpetrator of this event.
+- name: error.custom
+  type: flattened
+  description: Custom error context.
+- name: error.exception.attributes
+  type: flattened
+  description: Arbitrary attributes associated with the exception.
 - name: error.exception.code
   type: keyword
   description: The error code set when the error happened, e.g. database error code.
@@ -18,6 +24,10 @@
 - name: error.exception.module
   type: keyword
   description: The module namespace of the original error.
+- name: error.exception.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
 - name: error.exception.type
   type: keyword
   description: The type of the original error, e.g. the Java exception class name.
@@ -42,18 +52,38 @@
   type: keyword
   description: |
     A parametrized message. E.g. 'Could not connect to %s'. The property message is still required, and should be equal to the param_message, but with placeholders replaced. In some situations the param_message is used to group errors together.
+- name: error.log.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: kubernetes.namespace
   type: keyword
   description: |

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -56,18 +56,35 @@
   type: keyword
   description: |
     The lambda function version.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  index: false
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: http.response.transfer_size
   type: long
   unit: byte
@@ -221,10 +238,26 @@
   type: long
   description: |
     Sum of the durations of the compressed spans, in microseconds.
+- name: span.db.instance
+  type: keyword
+  description: |
+    Database instance.
 - name: span.db.link
   type: keyword
   description: |
     Database link.
+- name: span.db.type
+  type: keyword
+  description: |
+    Database type.
+- name: span.db.user.name
+  type: keyword
+  description: |
+    Database username.
+- name: span.db.statement
+  type: keyword
+  description: |
+    Database statement.
 - name: span.db.rows_affected
   type: long
   description: |
@@ -245,6 +278,18 @@
   type: long
   description: |
     Duration of the span, in microseconds.
+- name: span.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: span.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: span.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: span.message.age.ms
   type: long
   description: |
@@ -257,6 +302,10 @@
   type: keyword
   description: |
     Generic designation of a span in the scope of a transaction.
+- name: span.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the span.
 - name: span.subtype
   type: keyword
   description: |
@@ -279,6 +328,9 @@
   type: long
   description: |
     Timestamp of the event in microseconds since Unix epoch.
+- name: transaction.custom
+  type: flattened
+  description: Custom transaction context.
 - name: transaction.duration.us
   type: long
   description: |
@@ -306,6 +358,18 @@
   description: |
     A user-defined mapping of groups of marks in milliseconds.
   dynamic: true
+- name: transaction.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: transaction.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: transaction.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: transaction.message.age.ms
   type: long
   description: |
@@ -332,6 +396,9 @@
 - name: transaction.span_count.dropped
   type: long
   description: The total amount of dropped spans for this transaction.
+- name: transaction.span_count.started
+  type: long
+  description: The total amount of started spans for this transaction.
 - name: transaction.type
   type: keyword
   description: |

changelogs/head.asciidoc

@@ -5,6 +5,7 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 
 [float]
 ==== Breaking Changes
+- The unsupported apm_data_stream_migration ingest pipeline has been removed {pull}12102[12102].
 
 [float]
 ==== Bug fixes
@@ -20,4 +21,5 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 ==== Added
 - Update container base image to Ubuntu 22.04 {pull}11778[11778]
 - Add support for returning partial success response in OTLP input {pull}11883[11883]
-- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
+- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
+- Field mappings have been added for various formerly unindexed fields {pull}12102[12102]

docker-compose.yml

@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-ec4853e4-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.12.0-ec4853e4-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.12.0-ec4853e4-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

docs/integrations-index.asciidoc

@@ -86,4 +86,6 @@ include::upgrading.asciidoc[]
 
 include::release-notes.asciidoc[leveloffset=+1]
 
+include::known-issues.asciidoc[leveloffset=+1]
+
 include::{docdir}/redirects.asciidoc[]

docs/known-issues.asciidoc

@@ -0,0 +1,48 @@
+[[known-issues]]
+= Known issues
+
+APM has the following known issues:
+
+*Ingesting new JVM metrics in 8.9 and 8.10 breaks upgrade to 8.11 and stops ingestion* +
+_APM Server versions: 8.11.0, 8.11.1_ +
+_Elastic APM Java Agent versions: 1.39.0+_
+
+// Describe the conditions in which this issue occurs
+If you're using the Elastic APM Java Agent v1.39.0+ to send new JVM metrics to APM Server v8.9.x and v8.10.x,
+// Describe the behavior of the issue
+upgrading to 8.11.0 or 8.11.1 will silently fail and stop ingesting APM metrics.
+// Describe why it happens
+// This happens because...
+
+// Include exact error messages linked to this issue
+// so users searching for the error message end up here.
+After upgrading, you will see the following errors:
+
+* APM Server error logs:
++
+[source,txt]
+----
+failed to index document in 'metrics-apm.internal-default' (fail_processor_exception): Document produced by APM Server v8.11.1, which is newer than the installed APM integration (v8.10.3-preview-1695284222). The APM integration must be upgraded.
+----
+
+* Fleet error on integration package upgrade:
++
+[source,txt]
+----
+Failed installing package [apm] due to error: [ResponseError: mapper_parsing_exception
+	Root causes:
+		mapper_parsing_exception: Field [jvm.memory.non_heap.pool.committed] attempted to shadow a time_series_metric]
+----
+
+// Link to fix?
+A fix for this issue is in progress: https://github.com/elastic/kibana/pull/171712[elastic/kibana#171712].
+
+
+// TEMPLATE
+
+////
+*Brief description* +
+_Versions: XX.XX.XX, YY.YY.YY, ZZ.ZZ.ZZ_
+
+Detailed description.
+////