Merge branch 'main' into issue-7840

NOTICE.txt

@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20231121082709-886d078ba1f0
+Version: v7.0.0-alpha2.0.20231128050735-284683dcc123
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231121082709-886d078ba1f0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231128050735-284683dcc123/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.8.1
+Version: v0.8.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

apmpackage/apm/changelog.yml

@@ -1,8 +1,11 @@
 - version: generated
   changes:
-    - description: Placeholder
+    - description: Add missing mappings for various fields
       type: enhancement
-      link: https://github.com/elastic/apm-server/pull/123
+      link: https://github.com/elastic/apm-server/pull/12102
+    - description: Remove unsupported 7.x data stream migration pipeline
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/12102
 - version: 8.11.0
   changes:
     - description: Add geoip processing to add client.geo.* fields for app_logs

apmpackage/apm/data_stream/error_logs/fields/fields.yml

@@ -6,6 +6,12 @@
 - name: error.culprit
   type: keyword
   description: Function call which was the primary perpetrator of this event.
+- name: error.custom
+  type: flattened
+  description: Custom error context.
+- name: error.exception.attributes
+  type: flattened
+  description: Arbitrary attributes associated with the exception.
 - name: error.exception.code
   type: keyword
   description: The error code set when the error happened, e.g. database error code.
@@ -18,6 +24,10 @@
 - name: error.exception.module
   type: keyword
   description: The module namespace of the original error.
+- name: error.exception.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
 - name: error.exception.type
   type: keyword
   description: The type of the original error, e.g. the Java exception class name.
@@ -42,18 +52,38 @@
   type: keyword
   description: |
     A parametrized message. E.g. 'Could not connect to %s'. The property message is still required, and should be equal to the param_message, but with placeholders replaced. In some situations the param_message is used to group errors together.
+- name: error.log.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: kubernetes.namespace
   type: keyword
   description: |

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -56,18 +56,35 @@
   type: keyword
   description: |
     The lambda function version.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  index: false
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: http.response.transfer_size
   type: long
   unit: byte
@@ -221,10 +238,26 @@
   type: long
   description: |
     Sum of the durations of the compressed spans, in microseconds.
+- name: span.db.instance
+  type: keyword
+  description: |
+    Database instance.
 - name: span.db.link
   type: keyword
   description: |
     Database link.
+- name: span.db.type
+  type: keyword
+  description: |
+    Database type.
+- name: span.db.user.name
+  type: keyword
+  description: |
+    Database username.
+- name: span.db.statement
+  type: keyword
+  description: |
+    Database statement.
 - name: span.db.rows_affected
   type: long
   description: |
@@ -245,6 +278,18 @@
   type: long
   description: |
     Duration of the span, in microseconds.
+- name: span.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: span.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: span.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: span.message.age.ms
   type: long
   description: |
@@ -257,6 +302,10 @@
   type: keyword
   description: |
     Generic designation of a span in the scope of a transaction.
+- name: span.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the span.
 - name: span.subtype
   type: keyword
   description: |
@@ -279,6 +328,9 @@
   type: long
   description: |
     Timestamp of the event in microseconds since Unix epoch.
+- name: transaction.custom
+  type: flattened
+  description: Custom transaction context.
 - name: transaction.duration.us
   type: long
   description: |
@@ -306,6 +358,18 @@
   description: |
     A user-defined mapping of groups of marks in milliseconds.
   dynamic: true
+- name: transaction.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: transaction.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: transaction.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: transaction.message.age.ms
   type: long
   description: |
@@ -332,6 +396,9 @@
 - name: transaction.span_count.dropped
   type: long
   description: The total amount of dropped spans for this transaction.
+- name: transaction.span_count.started
+  type: long
+  description: The total amount of started spans for this transaction.
 - name: transaction.type
   type: keyword
   description: |

changelogs/head.asciidoc

@@ -5,6 +5,7 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 
 [float]
 ==== Breaking Changes
+- The unsupported apm_data_stream_migration ingest pipeline has been removed {pull}12102[12102].
 
 [float]
 ==== Bug fixes
@@ -20,4 +21,5 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 ==== Added
 - Update container base image to Ubuntu 22.04 {pull}11778[11778]
 - Add support for returning partial success response in OTLP input {pull}11883[11883]
-- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
+- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]
+- Field mappings have been added for various formerly unindexed fields {pull}12102[12102]

docker-compose.yml

@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-d6c4164b-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.12.0-d6c4164b-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.12.0-7521d760-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.12.0-d6c4164b-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -10,10 +10,10 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
 	github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231128050735-284683dcc123
 	github.com/elastic/elastic-agent-client/v7 v7.4.0
 	github.com/elastic/elastic-agent-libs v0.6.2
-	github.com/elastic/elastic-agent-system-metrics v0.8.1
+	github.com/elastic/elastic-agent-system-metrics v0.8.2
 	github.com/elastic/gmux v0.3.1
 	github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35
 	github.com/elastic/go-elasticsearch/v8 v8.10.0

go.sum

@@ -134,8 +134,8 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
 github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5 h1:URlDi0TZetSAo8KAIbaMZQhM5EG8+SDgs0jTen7WvMg=
 github.com/elastic/apm-data v0.1.1-0.20231019120308-82201b991ae5/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0 h1:ZVQuk/GSUNf/QvlC1bnI5EqXhBTpgrisbvQPT1S7p3w=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231121082709-886d078ba1f0/go.mod h1:eIy7raRUU0xxzyz5Kz6CWyeed8l7L0TYdHxXwsJKzT8=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231128050735-284683dcc123 h1:AQDBme+9nT+RrNqZcoRfqPUQE/gSzV1oML104+6fms8=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231128050735-284683dcc123/go.mod h1:E8InL/F98oNrwxO5Fu2ZgtCLkMGshVYNrXyvIyxJWko=
 github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA=
 github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M=
 github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo=
@@ -144,8 +144,8 @@ github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3
 github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
-github.com/elastic/elastic-agent-system-metrics v0.8.1 h1:eg6actuLeGJlIJFotHRdlAsz/3WhX2G8E0qI301IKBA=
-github.com/elastic/elastic-agent-system-metrics v0.8.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
+github.com/elastic/elastic-agent-system-metrics v0.8.2 h1:99H1o+WTD/g6jcQr1hUOZ+dsZkZF0SfNePmjGg7TnhU=
+github.com/elastic/elastic-agent-system-metrics v0.8.2/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
 github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
 github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=

systemtest/agentconfig_test.go

@@ -139,7 +139,7 @@ func TestAgentConfig(t *testing.T) {
 	require.Len(t, result.Hits.Hits, 2)
 	etag := gjson.GetBytes(result.Hits.Hits[0].RawSource, "labels.etag")
 	assert.Equal(t, etag1, strconv.Quote(etag.String()))
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits, "@timestamp", "labels.etag")
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits, "@timestamp", "labels.etag")
 }
 
 func queryAgentConfig(t testing.TB, serverURL, serviceName, serviceEnvironment, etag string) (map[string]string, *http.Response, map[string]interface{}) {