Merge branch 'main' into obs-docs-issue-2206

.buildkite/package.yml

@@ -1,6 +1,8 @@
 notify:
   - slack: "#apm-server"
     if: build.state == "failed"
+  - slack: "#observablt-bots"
+    if: "build.state != 'passed'"
 
 env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"

.github/dependabot.yml

@@ -0,0 +1,42 @@
+version: 2
+updates:
+
+  # GitHub actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    reviewers:
+      - "elastic/observablt-ci"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+
+  # GitHub composite actions
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows/generate-smoke-tests-list"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    reviewers:
+      - "elastic/observablt-ci"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows/setup-cluster-env"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    reviewers:
+      - "elastic/observablt-ci"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/benchmarks.yml

@@ -44,13 +44,13 @@ jobs:
       TF_VAR_REPO: ${{ github.repository }}
       GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
 
-      - uses: rlespinasse/github-slug-action@d1ca8ffbce40ea2371e35bca356db0acacf567d2
+      - uses: rlespinasse/github-slug-action@00d4236bb38fcea6fc4aa6b1dacc7187fc8e38a0
 
       - name: Set up env
         run: |
@@ -67,7 +67,7 @@ jobs:
             echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV"
           fi
 
-      - uses: hashicorp/vault-action@v2.6.0
+      - uses: hashicorp/vault-action@v2.7.4
         env:
           AWS_CREDENTIALS_PATH: secret/observability-team/ci/elastic-observability-aws-account-auth
           BENCHMARK_CLOUD_CREDENTIALS_PATH: secret/observability-team/ci/benchmark-cloud
@@ -97,7 +97,7 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-      - uses: hashicorp/setup-terraform@v2
+      - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.3.7
           terraform_wrapper: false
@@ -166,7 +166,7 @@ jobs:
         uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
         with:
           message: |
-            Nightly benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
+            Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
           vaultUrl: ${{ secrets.VAULT_ADDR }}
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
           vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
@@ -187,7 +187,7 @@ jobs:
                         "type": "section",
                         "text": {
                             "type": "mrkdwn",
-                            "text": "${{ github.event_name == 'schedule' && 'Nightly succesfully executed!' || 'Benchmarks succesfully executed!' }}"
+                            "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!"
                         },
                         "accessory": {
                             "type": "button",

.github/workflows/bump-elastic-stack.yml

@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.filter.outputs.matrix) }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
 

.github/workflows/bump-golang.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current
         with:
           vaultUrl: ${{ secrets.VAULT_ADDR }}

.github/workflows/ci.yml

@@ -26,8 +26,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true
@@ -45,8 +45,8 @@ jobs:
         os: ['macos-latest', 'ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true
@@ -57,8 +57,8 @@ jobs:
   system-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: systemtest/go.mod
           cache: true

.github/workflows/smoke-tests-schedule.yml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - id: generate
         name: Generate matrix
         uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current

.github/workflows/terraform-fmt.yml

@@ -12,6 +12,6 @@ jobs:
   terraform-fmt:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: hashicorp/setup-terraform@v2
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
       - run: terraform fmt -recursive -check -diff

.github/workflows/update-beats.yml

@@ -25,8 +25,8 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.filter.outputs.matrix) }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
       - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current

.go-version

@@ -1 +1 @@
-1.21.3
+1.21.4

NOTICE.txt

@@ -362,11 +362,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/apm-data
-Version: v0.1.1-0.20231019084040-a0f7950e9775
+Version: v0.1.1-0.20231019120308-82201b991ae5
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20231019084040-a0f7950e9775/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].20231019120308-82201b991ae5/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20231101145342-57df903385a9
+Version: v7.0.0-alpha2.0.20231121082709-886d078ba1f0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231101145342-57df903385a9/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20231121082709-886d078ba1f0/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -1040,11 +1040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.7.0
+Version: v0.8.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.7.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.8.1/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

changelogs/8.10.asciidoc

@@ -47,6 +47,7 @@ Fix tail-based sampling discarding low throughput and low sample rate traces {pu
 - Aggregated metrics now consider global labels to be part of a service's identity, and high cardinality global labels may cause services to be obscured {pull}11386[11386]
 - Event protobuf encoding for tail-based sampling changed (to a more efficient encoding) for event timestamp and duration {pull}11386[11386]
 - Use object notation for data_stream fields {pull}11311[11311]
+- Removed `processor.event` and `processor.name` from the data model {pull}11028[11028]
 
 [float]
 ==== Bug fixes

changelogs/8.11.asciidoc

@@ -3,8 +3,17 @@
 
 https://github.com/elastic/apm-server/compare/8.10\...8.11[View commits]
 
+* <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
 
+[float]
+[[release-notes-8.11.1]]
+=== APM version 8.11.1
+
+https://github.com/elastic/apm-server/compare/v8.11.0\...v8.11.1[View commits]
+
+No significant changes.
+
 [float]
 [[release-notes-8.11.0]]
 === APM version 8.11.0

changelogs/head.asciidoc

@@ -20,3 +20,4 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits]
 ==== Added
 - Update container base image to Ubuntu 22.04 {pull}11778[11778]
 - Add support for returning partial success response in OTLP input {pull}11883[11883]
+- Setting event timestamp from OTel observed timestamp when needed {pull}11935[11935]

docker-compose.yml

@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-0fcb9daf-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-7521d760-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.12.0-0fcb9daf-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.12.0-7521d760-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.12.0-0fcb9daf-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.12.0-7521d760-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

docs/configure/outputs/logstash.asciidoc

@@ -54,7 +54,7 @@ include::../../shared-kibana-endpoint.asciidoc[tag=shared-kibana-config]
 === {ls} configuration pipeline
 
 Finally, you must create a {ls} configuration pipeline that listens for incoming
-APM Server connections, dedots the `data_stream.*` fields, and indexes received events into {es}.
+APM Server connections and indexes received events into {es}.
 
 . Use the {logstash-ref}/plugins-inputs-elastic_agent.html[Elastic Agent input plugin] to configure
 {ls} to receive events from the APM Server. A minimal `input` config might look like this:
@@ -68,54 +68,6 @@ input {
 }
 ----
 
-. Use the {logstash-ref}/plugins-filters-mutate.html[Mutate filter plugin] to set up <<apm-data-streams,data streams>>.
-Because the {ls} {es} output doesn't understand dotted field notation, you must use this filter to
-dedot the default `data_stream.*` fields sent from APM Server to {ls}.
-+
-[source,conf]
-----
-filter {
-  mutate {
-    rename => {
-      "[data_stream.type]" => "[data_stream][type]"
-      "[data_stream.dataset]" => "[data_stream][dataset]"
-      "[data_stream.namespace]" => "[data_stream][namespace]"
-    }
-  }
-}
-----
-+
-.Expand to learn more
-[%collapsible]
-====
-****
-APM Server sends data stream information to {ls} in the following format:
-
-[source,json]
-----
-{
-  "data_stream.dataset": "apm",
-  "data_stream.type": "traces",
-  "data_stream.namespace": "default"
-}
-----
-
-{es} expects to receive data stream information in the following format:
-
-[source,json]
-----
-"data_stream" {
-  "dataset": "apm",
-  "type": "traces",
-  "dataset": "default"
-}
-----
-
-The mutation defined above transforms what APM Server sends to {ls} into a data format that {es} understands.
-This allows you to automatically route APM data to the appropriate data streams.
-****
-====
-
 . Use the {logstash-ref}/plugins-outputs-elasticsearch.html[{es} output plugin] to send
 events to {es} for indexing. A minimal `output` config might look like this:
 +
@@ -142,16 +94,6 @@ input {
   }
 }
 
-filter {
-  mutate {
-    rename => {
-      "[data_stream.type]" => "[data_stream][type]"
-      "[data_stream.dataset]" => "[data_stream][dataset]"
-      "[data_stream.namespace]" => "[data_stream][namespace]"
-    }
-  }
-}
-
 output {
   elasticsearch {
     data_stream => "true"
@@ -184,7 +126,7 @@ APM Server sends the following `@metadata` to {ls}:
 <2> The current version of APM Server.
 
 In addition to `@metadata`, APM Server provides other potentially useful fields, like the
-`processor.event` field and `data_stream` fields, which can be used to conditionally operate on
+`data_stream` field, which can be used to conditionally operate on
 {apm-guide-ref}/data-model.html[event types], namespaces, or datasets.
 
 As an example, you might want to use {ls} to route all `metric` events to the same custom metrics data stream,
@@ -194,7 +136,7 @@ rather than to service-specific data streams:
 ----
 output {
   if [@metadata][beat] == "apm-server" { <1>
-    if [processor][event] == "metric" { <2>
+    if [data_stream][type] == "metrics" { <2>
       elasticsearch {
         index => "%{[data_stream][type]}-custom-%{[data_stream][namespace]}" <3>
         action => "create" <4>

docs/data-model.asciidoc

@@ -236,9 +236,7 @@ and {apm-go-ref-v}/metrics.html[Go runtime] metrics in the Go Agent.
 Infrastructure and application metrics are important sources of information when debugging production systems,
 which is why we've made it easy to filter metrics for specific hosts or containers in the {kib} {kibana-ref}/metrics.html[metrics overview].
 
-Metrics have the `processor.event` property set to `metric`.
-
-TIP: Most agents limit keyword fields (e.g. `processor.event`) to 1024 characters,
+TIP: Most agents limit keyword fields to 1024 characters,
 non-keyword fields (e.g. `system.memory.total`) to 10,000 characters.
 
 Metrics are stored in metric indices.