apmpackage: use package spec v3 (#11822)

* wip: update package manifests * chore: resolve validation errors * fix: resolve validation error for dynamic_templates * fix: add object_type to fields type object * chore: use [email protected] * chore: update package-spec version to 3.0.1 * Update apmpackage/apm/changelog.yml Co-authored-by: Carson Ip <[email protected]> --------- Co-authored-by: Carson Ip <[email protected]>
elastic · Jan 12, 2024 · 8a1d41a · 8a1d41a
1 parent 1b36392
commit 8a1d41a
Show file tree

Hide file tree

Showing 27 changed files with 290 additions and 260 deletions.
diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml
@@ -1,8 +1,8 @@
 - version: generated
   changes:
-    - description: Placeholder
+    - description: Migrate to package-spec v3
       type: enhancement
-      link: https://github.com/elastic/apm-server/pull/123
+      link: https://github.com/elastic/apm-server/pull/11822
 - version: 8.12.0
   changes:
     - description: Add missing mappings for various fields

diff --git a/apmpackage/apm/data_stream/app_logs/fields/fields.yml b/apmpackage/apm/data_stream/app_logs/fields/fields.yml
@@ -5,6 +5,7 @@
     The way the agent is installed / used.
 - name: http.request.headers
   type: object
+  object_type: keyword
   description: |
     The canonical headers of the monitored HTTP request.
 - name: http.response.finished
@@ -13,6 +14,7 @@
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
   type: object
+  object_type: keyword
   description: |
     The canonical headers of the monitored HTTP response.
 - name: kubernetes.namespace
@@ -94,6 +96,8 @@
     The lambda function version.
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
+  scaling_factor: 1000000
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
diff --git a/apmpackage/apm/data_stream/app_logs/manifest.yml b/apmpackage/apm/data_stream/app_logs/manifest.yml
@@ -14,9 +14,3 @@ elasticsearch:
       # ad-hoc searches on HTTP request headers without incurring storage cost
       # for users who do not need this capability.
       dynamic: false
-      dynamic_templates:
-        - numeric_labels:
-            path_match: numeric_labels.*
-            mapping:
-              type: scaled_float
-              scaling_factor: 1000000
diff --git a/apmpackage/apm/data_stream/app_metrics/fields/fields.yml b/apmpackage/apm/data_stream/app_metrics/fields/fields.yml
@@ -61,6 +61,8 @@
     Version of the runtime used.
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
+  scaling_factor: 1000000
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
diff --git a/apmpackage/apm/data_stream/app_metrics/manifest.yml b/apmpackage/apm/data_stream/app_metrics/manifest.yml
@@ -7,16 +7,15 @@ elasticsearch:
   # Ensures agents have permissions to write data to `metrics-*-*`
   dynamic_dataset: true
   dynamic_namespace: true
+  # Individual measurements are typically uninteresting, so
+  # use synthetic source to reduce storage size.
+  source_mode: synthetic
   index_template:
     mappings:
       # Application metrics must be dynamically mapped,
       # as their names are application-specific and not
       # known ahead of time.
       dynamic: true
-      # Individual measurements are typically uninteresting, so
-      # use synthetic source to reduce storage size.
-      _source:
-        mode: synthetic
       # Install dynamic templates for use in dynamically
       # mapping complex application metrics.
       dynamic_templates:
@@ -34,8 +33,3 @@ elasticsearch:
             mapping:
               type: double
               index: false
-        - numeric_labels:
-            path_match: numeric_labels.*
-            mapping:
-              type: scaled_float
-              scaling_factor: 1000000
diff --git a/apmpackage/apm/data_stream/error_logs/fields/fields.yml b/apmpackage/apm/data_stream/error_logs/fields/fields.yml
@@ -62,16 +62,19 @@
     The original body of the monitored HTTP request.
 - name: http.request.env
   type: object
+  object_type: keyword
   dynamic: true
   description: |
     The CGI-like environment variables of the monitored HTTP request.
 - name: http.request.cookies
   type: object
+  object_type: keyword
   dynamic: true
   description: |
     The cookies of the monitored HTTP request.
 - name: http.request.headers
   type: object
+  object_type: keyword
   dynamic: true
   description: |
     The canonical headers of the monitored HTTP request.
@@ -81,6 +84,7 @@
     Used by the Node agent to indicate when in the response life cycle an error has occurred.
 - name: http.response.headers
   type: object
+  object_type: keyword
   dynamic: true
   description: |
     The canonical headers of the monitored HTTP response.
@@ -187,6 +191,8 @@
     Keyword of specific relevance in the service's domain (eg. 'request', 'backgroundjob', etc)
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
+  scaling_factor: 1000000
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
diff --git a/apmpackage/apm/data_stream/error_logs/manifest.yml b/apmpackage/apm/data_stream/error_logs/manifest.yml
@@ -11,9 +11,3 @@ elasticsearch:
       # ad-hoc searches on HTTP request headers without incurring storage cost
       # for users who do not need this capability.
       dynamic: false
-      dynamic_templates:
-        - numeric_labels:
-            path_match: numeric_labels.*
-            mapping:
-              type: scaled_float
-              scaling_factor: 1000000
diff --git a/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml
@@ -163,6 +163,20 @@
     Keyword of specific relevance in the service's domain (eg. 'request', 'backgroundjob', etc)
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
+  scaling_factor: 1000000
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
+- name: transaction.duration.summary
+  type: aggregate_metric_double
+  metrics: ["sum", "value_count"]
+  default_metric: "sum"
+  description: |
+    Total duration of transactions. This field is used for efficient average aggregation on the field
+- name: event.success_count
+  type: aggregate_metric_double
+  metrics: ["sum", "value_count"]
+  default_metric: "sum"
+  description: |
+    Transaction success counts. This field is used for efficient average aggregation on the field
diff --git a/apmpackage/apm/data_stream/internal_metrics/manifest.yml b/apmpackage/apm/data_stream/internal_metrics/manifest.yml
@@ -4,35 +4,16 @@ dataset: apm.internal
 ilm_policy: metrics-apm.internal_metrics-default_policy
 elasticsearch:
   dynamic_namespace: true
+  # Individual measurements are typically uninteresting, so
+  # use synthetic source to reduce storage size.
+  source_mode: synthetic
   index_template:
     mappings:
       # We are in full control of the field names,
       # but do not block ingestion on unexpected fields.
       dynamic: runtime
-      # Individual measurements are typically uninteresting, so
-      # use synthetic source to reduce storage size.
-      _source:
-        mode: synthetic
-      # We map transaction.duration.summary here because aggregate_metric_double
-      # is not currently supported by package-spec.
-      #
-      # See https://github.com/elastic/package-spec/issues/363
-      properties:
-        transaction:
-          properties:
-            duration:
-              properties:
-                summary:
-                  type: aggregate_metric_double
-                  metrics: [sum, value_count]
-                  default_metric: sum
-        event:
-          properties:
-            success_count:
-              type: aggregate_metric_double
-              metrics: [sum, value_count]
-              default_metric: sum
     settings:
       index:
-        sort.field: "@timestamp"
-        sort.order: desc
+        sort:
+          field: "@timestamp"
+          order: desc
diff --git a/apmpackage/apm/data_stream/rum_traces/manifest.yml b/apmpackage/apm/data_stream/rum_traces/manifest.yml
@@ -11,9 +11,3 @@ elasticsearch:
       # ad-hoc searches on HTTP request headers without incurring storage cost
       # for users who do not need this capability.
       dynamic: false
-      dynamic_templates:
-        - numeric_labels:
-            path_match: numeric_labels.*
-            mapping:
-              type: scaled_float
-              scaling_factor: 1000000
diff --git a/apmpackage/apm/data_stream/service_destination_interval_metrics/fields/ecs.yml b/apmpackage/apm/data_stream/service_destination_interval_metrics/fields/ecs.yml
@@ -2,6 +2,9 @@
   name: agent.name
 - external: ecs
   name: event.outcome
+- external: ecs
+  name: labels
+  dynamic: true
 - external: ecs
   name: observer.hostname
 - external: ecs
@@ -14,6 +17,3 @@
   name: service.environment
 - external: ecs
   name: service.name
-- external: ecs
-  name: labels
-  dynamic: true
diff --git a/apmpackage/apm/data_stream/service_destination_interval_metrics/fields/fields.yml b/apmpackage/apm/data_stream/service_destination_interval_metrics/fields/fields.yml
@@ -37,6 +37,8 @@
   unit: micros
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
+  scaling_factor: 1000000
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
diff --git a/apmpackage/apm/data_stream/service_destination_interval_metrics/manifest.yml b/apmpackage/apm/data_stream/service_destination_interval_metrics/manifest.yml
@@ -4,18 +4,18 @@ dataset: apm.service_destination.{{ .Interval }}
 ilm_policy: metrics-apm.service_destination_{{ .Interval }}_metrics-default_policy
 elasticsearch:
   dynamic_namespace: true
+  # Individual measurements are typically uninteresting, so
+  # use synthetic source to reduce storage size.
+  source_mode: synthetic
   index_template:
     mappings:
       # We are in full control of the field names,
       # but do not block ingestion on unexpected fields.
       dynamic: runtime
-      # Individual measurements are typically uninteresting, so
-      # use synthetic source to reduce storage size.
-      _source:
-        mode: synthetic
     settings:
       index:
-        sort.field: "@timestamp"
-        sort.order: desc
+        sort:
+          field: "@timestamp"
+          order: desc
     data_stream:
       hidden: {{ .Hidden }}
diff --git a/apmpackage/apm/data_stream/service_summary_interval_metrics/fields/ecs.yml b/apmpackage/apm/data_stream/service_summary_interval_metrics/fields/ecs.yml
@@ -1,5 +1,8 @@
 - external: ecs
   name: agent.name
+- external: ecs
+  name: labels
+  dynamic: true
 - external: ecs
   name: observer.hostname
 - external: ecs
@@ -12,6 +15,3 @@
   name: service.environment
 - external: ecs
   name: service.name
-- external: ecs
-  name: labels
-  dynamic: true
diff --git a/apmpackage/apm/data_stream/service_summary_interval_metrics/fields/fields.yml b/apmpackage/apm/data_stream/service_summary_interval_metrics/fields/fields.yml
@@ -14,6 +14,7 @@
     Name of the programming language used.
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.

diff --git a/apmpackage/apm/data_stream/service_summary_interval_metrics/manifest.yml b/apmpackage/apm/data_stream/service_summary_interval_metrics/manifest.yml
@@ -9,13 +9,13 @@ elasticsearch:
       # We are in full control of the field names,
       # but do not block ingestion on unexpected fields.
       dynamic: runtime
-      # Individual measurements are typically uninteresting, so
-      # use synthetic source to reduce storage size.
-      _source:
-        mode: synthetic
     settings:
       index:
-        sort.field: "@timestamp"
-        sort.order: desc
+        sort:
+          field: "@timestamp"
+          order: desc
     data_stream:
       hidden: {{ .Hidden }}
+  # Individual measurements are typically uninteresting, so
+  # use synthetic source to reduce storage size.
+  source_mode: synthetic
diff --git a/apmpackage/apm/data_stream/service_transaction_interval_metrics/fields/ecs.yml b/apmpackage/apm/data_stream/service_transaction_interval_metrics/fields/ecs.yml
@@ -1,5 +1,8 @@
 - external: ecs
   name: agent.name
+- external: ecs
+  name: labels
+  dynamic: true
 - external: ecs
   name: observer.hostname
 - external: ecs
@@ -12,6 +15,3 @@
   name: service.environment
 - external: ecs
   name: service.name
-- external: ecs
-  name: labels
-  dynamic: true
diff --git a/apmpackage/apm/data_stream/service_transaction_interval_metrics/fields/fields.yml b/apmpackage/apm/data_stream/service_transaction_interval_metrics/fields/fields.yml
@@ -22,9 +22,22 @@
     Pre-aggregated histogram of transaction durations.
 - name: numeric_labels
   type: object
+  object_type: scaled_float
   dynamic: true
   description: |
     Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float.
 - name: service_transaction.aggregation.overflow_count
   type: long
   description: Number of aggregation groups that overflowed for service transaction metrics aggregation.
+- name: transaction.duration.summary
+  type: aggregate_metric_double
+  metrics: ["sum", "value_count"]
+  default_metric: "sum"
+  description: |
+    Total duration of transactions. This can be used to efficiently aggregate the average duration
+- name: event.success_count
+  type: aggregate_metric_double
+  metrics: ["sum", "value_count"]
+  default_metric: "sum"
+  description: |
+    Transaction success counts. This field is used for efficient average aggregation on the field
diff --git a/apmpackage/apm/data_stream/service_transaction_interval_metrics/manifest.yml b/apmpackage/apm/data_stream/service_transaction_interval_metrics/manifest.yml
@@ -4,37 +4,18 @@ dataset: apm.service_transaction.{{ .Interval }}
 ilm_policy: metrics-apm.service_transaction_{{ .Interval }}_metrics-default_policy
 elasticsearch:
   dynamic_namespace: true
+  # Individual measurements are typically uninteresting, so
+  # use synthetic source to reduce storage size.
+  source_mode: synthetic
   index_template:
     mappings:
       # We are in full control of the field names,
       # but do not block ingestion on unexpected fields.
       dynamic: runtime
-      # Individual measurements are typically uninteresting, so
-      # use synthetic source to reduce storage size.
-      _source:
-        mode: synthetic
-      # We map transaction.duration.summary here because aggregate_metric_double
-      # is not currently supported by package-spec.
-      #
-      # See https://github.com/elastic/package-spec/issues/363
-      properties:
-        transaction:
-          properties:
-            duration:
-              properties:
-                summary:
-                  type: aggregate_metric_double
-                  metrics: [sum, value_count]
-                  default_metric: sum
-        event:
-          properties:
-            success_count:
-              type: aggregate_metric_double
-              metrics: [sum, value_count]
-              default_metric: sum
     settings:
       index:
-        sort.field: "@timestamp"
-        sort.order: desc
+        sort:
+          field: "@timestamp"
+          order: desc
     data_stream:
       hidden: {{ .Hidden }}