chore(dev): add multiservice opentelemetry trace e2e test

.config/nextest.toml

@@ -20,3 +20,10 @@ slow-timeout = { period = "30s", terminate-after = 4 }
 # output test results at target/nextest/default/junit.xml
 path = "junit.xml"
 store-failure-output = true
+
+[[profile.default.overrides]]
+filter = 'test(opentelemetry::multiservice_traces::trace_round_trip)'
+# This test stops its producers and collectors, so a retry needs a fresh Compose run.
+# CLI retries override this setting. Run vdev with --retries 0 (its default is 2).
+retries = 0
+slow-timeout = { period = "30s", terminate-after = 10 }

.github/workflows/ci-integration-review.yml

@@ -187,10 +187,11 @@ jobs:
           command: vdev integration run ${{ matrix.service }} ${{ runner.debug == '1' && '--always-show-logs' || '' }}
 
   e2e-tests:
+    # The dedicated command also starts with the ordinary opentelemetry-traces command.
+    if: github.repository == 'vectordotdev/vector' && !startsWith(github.event.review.body, '/ci-run-e2e-opentelemetry-traces-multiservice')
     needs:
       - prep-pr
       - build-test-runner
-    if: github.repository == 'vectordotdev/vector'
     runs-on: ubuntu-24.04-8core
     timeout-minutes: 30
     permissions:
@@ -252,13 +253,78 @@ jobs:
           max_attempts: 3
           command: vdev e2e run ${{ matrix.service }} ${{ runner.debug == '1' && '--always-show-logs' || '' }}
 
+  otel-traces-multiservice:
+    name: Multiservice OpenTelemetry trace validation
+    if: github.repository == 'vectordotdev/vector' && startsWith(github.event.review.body, '/ci-run-e2e-opentelemetry-traces-multiservice')
+    needs:
+      - prep-pr
+      - build-test-runner
+    runs-on: ubuntu-24.04-8core
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      packages: read # Required to pull test runner image from GHCR
+      id-token: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          submodules: "recursive"
+          ref: ${{ github.event.review.commit_id }}
+
+      - uses: ./.github/actions/dd-token
+
+      - uses: ./.github/actions/setup
+        with:
+          vdev: true
+          datadog-ci: true
+
+      - name: Pull test runner image
+        uses: ./.github/actions/pull-test-runner
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ github.event.review.commit_id }}
+
+      - name: Multiservice OpenTelemetry trace validation
+        run: |
+          mkdir -p target/otel-traces-multiservice-evidence
+          set -o pipefail
+          vdev e2e run opentelemetry-traces-multiservice --retries 0 --always-show-logs 2>&1 | tee target/otel-traces-multiservice-evidence/runner.log
+
+      - name: Collect multiservice trace evidence
+        if: always()
+        run: |
+          mkdir -p target/otel-traces-multiservice-evidence
+          git rev-parse HEAD > target/otel-traces-multiservice-evidence/revision.txt
+          cp -R tests/e2e/opentelemetry-traces-multiservice/config tests/e2e/opentelemetry-traces-multiservice/data target/otel-traces-multiservice-evidence/
+          if docker volume inspect opentelemetry_traces_multiservice_evidence >/dev/null 2>&1; then
+            docker run --rm \
+              -v opentelemetry_traces_multiservice_evidence:/output/opentelemetry-traces-multiservice:ro \
+              -v "$PWD/target/otel-traces-multiservice-evidence:/evidence" \
+              alpine:3.20 sh -c 'if [ -d /output/opentelemetry-traces-multiservice ]; then cp -R /output/opentelemetry-traces-multiservice/. /evidence/; fi'
+          fi
+          summaries=(target/otel-traces-multiservice-evidence/*/summary.md)
+          if [ -f "${summaries[0]}" ]; then
+            cat "${summaries[@]}" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo 'Multiservice trace test did not produce a validation summary. See runner.log and the failed job steps.' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload multiservice trace evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: otel-traces-multiservice-${{ github.event.review.commit_id }}
+          path: target/otel-traces-multiservice-evidence/
+          retention-days: 14
+
   update-pr-status:
     name: Signal result to PR
     runs-on: ubuntu-24.04
     timeout-minutes: 5
     needs:
       - integration-tests
       - e2e-tests
+      - otel-traces-multiservice
     if: always() && (startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-e2e'))
     permissions:
       statuses: write # Required to set final commit status

Cargo.toml

@@ -1159,7 +1159,8 @@ e2e-tests-opentelemetry = [
   "sinks-console",
   "sinks-file",
   "codecs-opentelemetry",
-  "dep:prost-reflect"
+  "dep:prost-reflect",
+  "dep:bollard"
 ]
 
 vector-api-tests = [

tests/e2e/opentelemetry-traces-multiservice/README.md

@@ -0,0 +1,103 @@
+# Multiservice OpenTelemetry trace validation
+
+This manual E2E test checks that Vector preserves multilingual, multiservice
+traces from the
+[OpenTelemetry Demo 3.1.0](https://github.com/open-telemetry/opentelemetry-demo/tree/3.1.0)
+using the existing `vdev` Compose harness and
+[OTel Collector](https://opentelemetry.io/docs/collector/) 0.159.0.
+
+It provides realistic trace-preservation evidence for changes such as the
+[trace model RFC](../../../rfcs/2026-04-29-25329-trace-data-model.md).
+It runs manually because the full application setup is heavier than the
+fixture-based trace tests.
+
+## Input and destination
+
+```mermaid
+flowchart LR
+    demo["OpenTelemetry Demo"] --> ingress["OTel Collector (input)"]
+    ingress --> input["input.jsonl"]
+    ingress -->|"OTLP/HTTP or gRPC (separate sequential runs)"| source
+    subgraph vector["Vector"]
+        source["OpenTelemetry source"] -->|"otel.traces (no transforms)"| sink["OpenTelemetry sink"]
+    end
+    sink -->|"OTLP/HTTP"| egress["OTel Collector (output)"]
+    egress --> output["output.jsonl"]
+    subgraph validation["Rust validation"]
+        coverage["Required services and trace shapes"]
+        preservation["Compare spans, resources, scopes, schema URLs and duplicate counts"]
+    end
+    input -.-> coverage
+    input -.-> preservation
+    output -.-> preservation
+```
+
+Rust validates the captures after each run. The comparison allows batching, span order,
+and attribute-map order to differ.
+
+[scenario.py](data/scenario.py) runs one simulated shopper through three rounds of ad,
+recommendation, cart, and multi-item checkout requests. Checkout also exercises
+the Kafka accounting and fraud-detection consumers.
+
+[demo.flagd.json](data/demo.flagd.json) uses the Demo's default flags except
+`cartFailure: 100%`. This generates cart-emptying error spans while checkout
+continues to email and Kafka.
+
+The [ingress Collector](data/collector-source.yaml) writes `input.jsonl` and
+forwards traces to Vector. The [matrix](config/test.yaml) runs each transport
+separately with the same checks:
+
+- HTTP (`otlphttp`): `http://vector:4318/v1/traces`.
+- gRPC (`otlp`): `vector:4317`.
+
+Application logs and metrics are not forwarded to Vector.
+
+Vector runs [data/vector.yaml](data/vector.yaml), mounted as
+`/etc/vector/vector.yaml`: an `opentelemetry` source with
+`use_otlp_decoding: true` feeds `otel.traces` directly to an `opentelemetry`
+sink, which sends OTLP/HTTP to `http://otel-collector-sink:4318/v1/traces`.
+
+The destination is a [second local Collector](data/collector-sink.yaml) that
+writes `output.jsonl`.
+
+## Validations
+
+The [Rust test](../opentelemetry/multiservice_traces/mod.rs) verifies:
+
+- Completion: requires the workload marker, allows Kafka consumers to finish,
+  stops the input, and waits up to 120 seconds for Vector to drain.
+- Service coverage: requires spans from 14 services exercised by the workload.
+- Trace shapes: requires roots, multi-span traces, children with captured
+  parents, cross-service parent relationships, and error spans.
+- Preservation: compares every span and its resource, scope, and schema URLs.
+  Missing, extra, or changed spans and duplicate counts fail. Only batching,
+  span order, and attribute-map order may differ.
+- Capture format: fails on malformed IDs or unknown capture fields.
+
+## Coverage and limitations
+
+- Decoding: only `use_otlp_decoding: true` is tested.
+- Optional coverage: events, links, and scope groups with multiple trace IDs
+  are reported in `coverage.json` and compared when present. They are not required,
+  so a run without them can pass.
+- Edge cases: the workload does not guarantee all field values, such as byte
+  attributes, missing scopes, or nonzero dropped counts.
+
+## Running and results
+
+With Docker available, run both transports:
+
+```shell
+cargo vdev e2e run opentelemetry-traces-multiservice --retries 0 --always-show-logs
+```
+
+To run one transport, add `-e 0.159.0-otlphttp` (HTTP) or `-e 0.159.0-otlp` (gRPC).
+
+In CI, a Vector-team member can submit a PR review starting with
+`/ci-run-e2e-opentelemetry-traces-multiservice` to test the reviewed commit.
+Comments, “run all” commands, automatic PR CI, and schedules do not trigger it.
+
+Read the results on the CI job's summary page. Download the
+`otel-traces-multiservice-<commit>` artifact for input/output captures, `coverage.json`,
+`differences.txt`, configs, and runner logs. Results are separated by transport;
+failed runs may leave partial results.