systemtest: approve fields instead of _source

apmpackage/apm/data_stream/error_logs/fields/fields.yml

@@ -6,6 +6,12 @@
 - name: error.culprit
   type: keyword
   description: Function call which was the primary perpetrator of this event.
+- name: error.custom
+  type: flattened
+  description: Custom error context.
+- name: error.exception.attributes
+  type: flattened
+  description: Arbitrary attributes associated with the exception.
 - name: error.exception.code
   type: keyword
   description: The error code set when the error happened, e.g. database error code.
@@ -18,6 +24,10 @@
 - name: error.exception.module
   type: keyword
   description: The module namespace of the original error.
+- name: error.exception.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
 - name: error.exception.type
   type: keyword
   description: The type of the original error, e.g. the Java exception class name.
@@ -42,18 +52,38 @@
   type: keyword
   description: |
     A parametrized message. E.g. 'Could not connect to %s'. The property message is still required, and should be equal to the param_message, but with placeholders replaced. In some situations the param_message is used to group errors together.
+- name: error.log.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the error.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: kubernetes.namespace
   type: keyword
   description: |

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -56,18 +56,35 @@
   type: keyword
   description: |
     The lambda function version.
+- name: http.request.body
+  type: flattened
+  description: |
+    The original body of the monitored HTTP request.
+- name: http.request.env
+  type: flattened
+  description: |
+    The CGI-like environment variables of the monitored HTTP request.
+- name: http.request.cookies
+  type: flattened
+  description: |
+    The cookies of the monitored HTTP request.
 - name: http.request.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
   type: boolean
   description: |
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
-  type: object
+  type: flattened
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.headers_sent
+  type: boolean
+  index: false
+  description: |
+    Used by the Node agent to indicate whether HTTP response headers were sent.
 - name: http.response.transfer_size
   type: long
   unit: byte
@@ -221,10 +238,26 @@
   type: long
   description: |
     Sum of the durations of the compressed spans, in microseconds.
+- name: span.db.instance
+  type: keyword
+  description: |
+    Database instance.
 - name: span.db.link
   type: keyword
   description: |
     Database link.
+- name: span.db.type
+  type: keyword
+  description: |
+    Database type.
+- name: span.db.user.name
+  type: keyword
+  description: |
+    Database username.
+- name: span.db.statement
+  type: keyword
+  description: |
+    Database statement.
 - name: span.db.rows_affected
   type: long
   description: |
@@ -245,6 +278,18 @@
   type: long
   description: |
     Duration of the span, in microseconds.
+- name: span.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: span.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: span.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: span.message.age.ms
   type: long
   description: |
@@ -257,6 +302,10 @@
   type: keyword
   description: |
     Generic designation of a span in the scope of a transaction.
+- name: span.stacktrace
+  type: flattened
+  description: |
+    Stacktrace identifying the instrumented source code corresponding to the span.
 - name: span.subtype
   type: keyword
   description: |
@@ -279,6 +328,9 @@
   type: long
   description: |
     Timestamp of the event in microseconds since Unix epoch.
+- name: transaction.custom
+  type: flattened
+  description: Custom transaction context.
 - name: transaction.duration.us
   type: long
   description: |
@@ -306,6 +358,18 @@
   description: |
     A user-defined mapping of groups of marks in milliseconds.
   dynamic: true
+- name: transaction.message.body
+  type: keyword
+  description: The message body
+  index: false
+- name: transaction.message.routing_key
+  type: keyword
+  description: The message routing key
+  index: false
+- name: transaction.message.headers
+  type: flattened
+  description: The message headers
+  index: false
 - name: transaction.message.age.ms
   type: long
   description: |
@@ -332,6 +396,9 @@
 - name: transaction.span_count.dropped
   type: long
   description: The total amount of dropped spans for this transaction.
+- name: transaction.span_count.started
+  type: long
+  description: The total amount of started spans for this transaction.
 - name: transaction.type
   type: keyword
   description: |

systemtest/agentconfig_test.go

@@ -139,7 +139,7 @@ func TestAgentConfig(t *testing.T) {
 	require.Len(t, result.Hits.Hits, 2)
 	etag := gjson.GetBytes(result.Hits.Hits[0].RawSource, "labels.etag")
 	assert.Equal(t, etag1, strconv.Quote(etag.String()))
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits, "@timestamp", "labels.etag")
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits, "@timestamp", "labels.etag")
 }
 
 func queryAgentConfig(t testing.TB, serverURL, serviceName, serviceEnvironment, etag string) (map[string]string, *http.Response, map[string]interface{}) {

systemtest/aggregation_test.go

@@ -78,7 +78,7 @@ func TestTransactionAggregation(t *testing.T) {
 	result := estest.ExpectMinDocs(t, systemtest.Elasticsearch, 9, "metrics-apm.transaction*",
 		espoll.ExistsQuery{Field: "transaction.duration.histogram"},
 	)
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits)
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits)
 
 	// Make sure the _doc_count field is added such that aggregations return
 	// the appropriate per-bucket doc_count values.
@@ -140,7 +140,7 @@ func TestTransactionAggregationShutdown(t *testing.T) {
 	result := estest.ExpectMinDocs(t, systemtest.Elasticsearch, 3, "metrics-apm.transaction*",
 		espoll.ExistsQuery{Field: "transaction.duration.histogram"},
 	)
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits)
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits)
 }
 
 func TestServiceDestinationAggregation(t *testing.T) {
@@ -176,7 +176,16 @@ func TestServiceDestinationAggregation(t *testing.T) {
 	result := estest.ExpectDocs(t, systemtest.Elasticsearch, "metrics-apm.service_destination*",
 		espoll.ExistsQuery{Field: "span.destination.service.response_time.count"},
 	)
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits)
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits)
+
+	// _doc_count is not returned in fields, it is only visible in _source and
+	// in the results of aggregations.
+	//
+	// TODO(axw) we should use an aggregation, and check the resturned doc_counts.
+	for _, hit := range result.Hits.Hits {
+		docCount := hit.Source["_doc_count"].(float64)
+		assert.Equal(t, 5.0, docCount)
+	}
 }
 
 func TestTransactionAggregationLabels(t *testing.T) {
@@ -252,7 +261,16 @@ func TestServiceTransactionMetricsAggregation(t *testing.T) {
 	result := estest.ExpectMinDocs(t, systemtest.Elasticsearch, 2, "metrics-apm.service_transaction*",
 		espoll.TermQuery{Field: "metricset.name", Value: "service_transaction"},
 	)
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits)
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits)
+
+	// _doc_count is not returned in fields, it is only visible in _source and
+	// in the results of aggregations.
+	//
+	// TODO(axw) we should use an aggregation, and check the resturned doc_counts.
+	for _, hit := range result.Hits.Hits {
+		docCount := hit.Source["_doc_count"].(float64)
+		assert.Equal(t, 2.0, docCount)
+	}
 }
 
 func TestServiceTransactionMetricsAggregationLabels(t *testing.T) {
@@ -371,7 +389,7 @@ func TestServiceSummaryMetricsAggregation(t *testing.T) {
 	result := estest.ExpectDocs(t, systemtest.Elasticsearch, "metrics-apm.service_summary*",
 		espoll.TermQuery{Field: "metricset.name", Value: "service_summary"},
 	)
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits)
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits)
 }
 
 func TestServiceSummaryMetricsAggregationOverflow(t *testing.T) {
@@ -416,7 +434,7 @@ func TestServiceSummaryMetricsAggregationOverflow(t *testing.T) {
 		espoll.TermQuery{Field: "metricset.name", Value: "service_summary"},
 	)
 	// Ignore timestamp because overflow bucket uses time.Now()
-	approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits, "@timestamp")
+	approvaltest.ApproveFields(t, t.Name(), result.Hits.Hits, "@timestamp")
 }
 
 func TestNonDefaultRollupIntervalHiddenDataStream(t *testing.T) {