Merge branch 'main' into code-stacktrace

apmpackage/apm/changelog.yml

@@ -1,5 +1,11 @@
 - version: generated
   changes:
+    - description: Add missing mappings for various fields
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/12102
+    - description: Remove unsupported 7.x data stream migration pipeline
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/12102
     - description: Add span.code.stacktrace
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/12096

apmpackage/apm/data_stream/error_logs/fields/fields.yml

@@ -6,6 +6,12 @@
 - name: error.culprit
   type: keyword
   description: Function call which was the primary perpetrator of this event.
+- name: error.custom
+  type: flattened
+  description: Custom error context.
+- name: error.exception.attributes
+  type: flattened
+  description: Arbitrary attributes associated with the exception.
 - name: error.exception.code
   type: keyword
   description: The error code set when the error happened, e.g. database error code.
@@ -18,6 +24,10 @@
 - name: error.exception.module
   type: keyword
   description: The module namespace of the original error.
+- name: error.exception.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
 - name: error.exception.type
   type: keyword
   description: The type of the original error, e.g. the Java exception class name.
@@ -42,18 +52,38 @@
   type: keyword
   description: |
     A parametrized message. E.g. 'Could not connect to %s'. The property message is still required, and should be equal to the param_message, but with placeholders replaced. In some situations the param_message is used to group errors together.
+- name: error.log.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: kubernetes.namespace
   type: keyword
   description: |

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -56,18 +56,35 @@
   type: keyword
   description: |
     The lambda function version.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  index: false
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: http.response.transfer_size
   type: long
   unit: byte
@@ -221,10 +238,26 @@
   type: long
   description: |
     Sum of the durations of the compressed spans, in microseconds.
+- name: span.db.instance
+  type: keyword
+  description: |
+    Database instance.
 - name: span.db.link
   type: keyword
   description: |
     Database link.
+- name: span.db.type
+  type: keyword
+  description: |
+    Database type.
+- name: span.db.user.name
+  type: keyword
+  description: |
+    Database username.
+- name: span.db.statement
+  type: keyword
+  description: |
+    Database statement.
 - name: span.db.rows_affected
   type: long
   description: |
@@ -245,6 +278,18 @@
   type: long
   description: |
     Duration of the span, in microseconds.
+- name: span.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: span.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: span.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: span.message.age.ms
   type: long
   description: |
@@ -257,6 +302,10 @@
   type: keyword
   description: |
     Generic designation of a span in the scope of a transaction.
+- name: span.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the span.
 - name: span.subtype
   type: keyword
   description: |
@@ -283,6 +332,9 @@
   type: long
   description: |
     Timestamp of the event in microseconds since Unix epoch.
+- name: transaction.custom
+  type: flattened
+  description: Custom transaction context.
 - name: transaction.duration.us
   type: long
   description: |
@@ -310,6 +362,18 @@
   description: |
     A user-defined mapping of groups of marks in milliseconds.
   dynamic: true
+- name: transaction.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: transaction.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: transaction.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: transaction.message.age.ms
   type: long
   description: |
@@ -336,6 +400,9 @@
 - name: transaction.span_count.dropped
   type: long
   description: The total amount of dropped spans for this transaction.
+- name: transaction.span_count.started
+  type: long
+  description: The total amount of started spans for this transaction.
 - name: transaction.type
   type: keyword
   description: |

changelogs/head.asciidoc

@@ -5,6 +5,7 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 
 [float]
 ==== Breaking Changes
+- The unsupported apm_data_stream_migration ingest pipeline has been removed {pull}12102[12102].
 
 [float]
 ==== Bug fixes
@@ -21,4 +22,5 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 - Update container base image to Ubuntu 22.04 {pull}11778[11778]
 - Add support for returning partial success response in OTLP input {pull}11883[11883]
 - Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
-- Add support for Otel code.stacktrace {pull}12096[12096]
+- Field mappings have been added for various formerly unindexed fields {pull}12102[12102]
+- Add support for Otel code.stacktrace {pull}12096[12096]

docs/configure/outputs/logstash.asciidoc

@@ -54,7 +54,7 @@ include::../../shared-kibana-endpoint.asciidoc[tag=shared-kibana-config]
 === {ls} configuration pipeline
 
 Finally, you must create a {ls} configuration pipeline that listens for incoming
-APM Server connections, dedots the `data_stream.*` fields, and indexes received events into {es}.
+APM Server connections and indexes received events into {es}.
 
 . Use the {logstash-ref}/plugins-inputs-elastic_agent.html[Elastic Agent input plugin] to configure
 {ls} to receive events from the APM Server. A minimal `input` config might look like this:
@@ -68,54 +68,6 @@ input {
 }
 ----
 
-. Use the {logstash-ref}/plugins-filters-mutate.html[Mutate filter plugin] to set up <<apm-data-streams,data streams>>.
-Because the {ls} {es} output doesn't understand dotted field notation, you must use this filter to
-dedot the default `data_stream.*` fields sent from APM Server to {ls}.
-+
-[source,conf]
-----
-filter {
-  mutate {
-    rename => {
-      "[data_stream.type]" => "[data_stream][type]"
-      "[data_stream.dataset]" => "[data_stream][dataset]"
-      "[data_stream.namespace]" => "[data_stream][namespace]"
-    }
-  }
-}
-----
-+
-.Expand to learn more
-[%collapsible]
-====
-****
-APM Server sends data stream information to {ls} in the following format:
-
-[source,json]
-----
-{
-  "data_stream.dataset": "apm",
-  "data_stream.type": "traces",
-  "data_stream.namespace": "default"
-}
-----
-
-{es} expects to receive data stream information in the following format:
-
-[source,json]
-----
-"data_stream" {
-  "dataset": "apm",
-  "type": "traces",
-  "dataset": "default"
-}
-----
-
-The mutation defined above transforms what APM Server sends to {ls} into a data format that {es} understands.
-This allows you to automatically route APM data to the appropriate data streams.
-****
-====
-
 . Use the {logstash-ref}/plugins-outputs-elasticsearch.html[{es} output plugin] to send
 events to {es} for indexing. A minimal `output` config might look like this:
 +
@@ -142,16 +94,6 @@ input {
   }
 }
 
-filter {
-  mutate {
-    rename => {
-      "[data_stream.type]" => "[data_stream][type]"
-      "[data_stream.dataset]" => "[data_stream][dataset]"
-      "[data_stream.namespace]" => "[data_stream][namespace]"
-    }
-  }
-}
-
 output {
   elasticsearch {
     data_stream => "true"