functionaltests: add first test case (#14935) (#15284)

Implement first test verifying upgrade from 8.15.4 to 8.16.0 for a fresh Elastic Cloud deployment.

Provides a new folder structure and helpers to quickly implement functional tests for APM Server on Elastic Cloud.

---------

Co-authored-by: Victor Martinez <[email protected]>
(cherry picked from commit da158f0)

Co-authored-by: Edoardo Tenani <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/workflows/functional-tests.yml

@@ -0,0 +1,66 @@
+---
+name: functional-tests
+
+on:
+  workflow_dispatch: ~
+  schedule:
+    - cron: '0 3 * * 1-5'
+
+permissions:
+  contents: read
+  id-token: write
+
+env:
+  TF_VAR_BRANCH: ${{ github.ref_name }}
+  TF_VAR_BUILD_ID: ${{ github.run_id }}
+  TF_VAR_ENVIRONMENT: 'ci'
+  TF_VAR_REPO: ${{ github.repository }}
+  TERRAFORM_VERSION: 1.10.2
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - 'qa'
+          - 'pro'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: "${{ env.TERRAFORM_VERSION }}"
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'functionaltests/go.mod'
+
+      - uses: elastic/oblt-actions/google/auth@v1
+
+      - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
+        with:
+          export_to_environment: true
+          secrets: |-
+            EC_API_KEY:elastic-observability/elastic-cloud-observability-team-${{ matrix.environment }}-api-key
+
+      - run: |
+          export TF_VAR_CREATED_DATE=$(date +%s)
+          cd functionaltests && go test -v -timeout=20m -target "${{ matrix.environment }}" ./
+
+  notify:
+    if: always()
+    runs-on: ubuntu-latest
+    needs:
+      - run
+    steps:
+      - id: check
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+      - uses: elastic/oblt-actions/slack/notify-result@v1
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
+          status: ${{ steps.check.outputs.status }}

functionaltests/8_15_test.go

@@ -0,0 +1,152 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package functionaltests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/elastic/apm-server/functionaltests/internal/esclient"
+	"github.com/elastic/apm-server/functionaltests/internal/gen"
+	"github.com/elastic/apm-server/functionaltests/internal/terraform"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
+	ecAPICheck(t)
+
+	start := time.Now()
+	ctx := context.Background()
+
+	t.Log("creating deploment with terraform")
+	tf, err := terraform.New(t, t.Name())
+	require.NoError(t, err)
+	ecTarget := terraform.Var("ec_target", *target)
+	ecRegion := terraform.Var("ec_region", regionFrom(*target))
+	version := terraform.Var("stack_version", "8.15.4")
+	name := terraform.Var("name", t.Name())
+	require.NoError(t, tf.Apply(ctx, ecTarget, ecRegion, version, name))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Cleanup(func() {
+		if !t.Failed() || (t.Failed() && *cleanupOnFailure) {
+			t.Log("cleanup terraform resources")
+			require.NoError(t, tf.Destroy(ctx, ecTarget, ecRegion, name, version))
+		} else {
+			t.Log("test failed and cleanup-on-failure is false, skipping cleanup")
+		}
+	})
+
+	var escfg esclient.Config
+	tf.Output("apm_url", &escfg.APMServerURL)
+	tf.Output("es_url", &escfg.ElasticsearchURL)
+	tf.Output("username", &escfg.Username)
+	tf.Output("password", &escfg.Password)
+	tf.Output("kb_url", &escfg.KibanaURL)
+
+	t.Logf("created deployment %s", escfg.KibanaURL)
+
+	ecc, err := esclient.New(escfg)
+	require.NoError(t, err)
+
+	t.Log("creating APM API key")
+	apikey, err := ecc.CreateAPMAPIKey(ctx, t.Name())
+	require.NoError(t, err)
+
+	g := gen.New(escfg.APMServerURL, apikey)
+	g.Logger = zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
+
+	previous, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
+
+	beforeUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	assertDocCount(t, beforeUpgradeCount, previous, expectedIngestForASingleRun())
+
+	t.Log("check data streams")
+	var dss []types.DataStream
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("upgrade to 8.16.0")
+	require.NoError(t, tf.Apply(ctx, ecTarget, ecRegion, name, terraform.Var("stack_version", "8.16.0")))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("check number of documents after upgrade")
+	afterUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	// We assert that no changes happened in the number of documents after upgrade
+	// to ensure the state didn't change before running the next ingestion round
+	// and further assertions.
+	// We don't expect any change here unless something broke during the upgrade.
+	assertDocCount(t, afterUpgradeCount, esclient.APMDataStreamsDocCount{}, beforeUpgradeCount)
+
+	t.Log("check data streams after upgrade, no rollover expected")
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
+
+	t.Log("check number of documents")
+	afterUpgradeIngestionCount, err := getDocsCountPerDS(t, ctx, ecc)
+	require.NoError(t, err)
+	assertDocCount(t, afterUpgradeIngestionCount, afterUpgradeCount, expectedIngestForASingleRun())
+
+	// Confirm datastreams are
+	// v managed by DSL if created after 8.15.0
+	// x managed by ILM if created before 8.15.0
+	t.Log("check data streams and verify lazy rollover happened")
+	dss2, err := ecc.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     2,
+		IndicesManagedBy: []string{"Data stream lifecycle", "Data stream lifecycle"},
+	}, dss2)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	res, err := ecc.GetESErrorLogs(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, res.Hits.Total.Value)
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/main.tf

@@ -0,0 +1,18 @@
+module "ec_deployment" {
+  source = "../../testing/infra/terraform/modules/ec_deployment"
+  region = var.ec_region
+
+  deployment_template    = "aws-storage-optimized"
+  deployment_name_prefix = var.name
+
+  // self monitoring is enabled so we can inspect Elasticsearch
+  // logs from tests.
+  observability_deployment = "self"
+
+  apm_server_size = "1g"
+
+  elasticsearch_size       = "4g"
+  elasticsearch_zone_count = 1
+
+  stack_version = var.stack_version
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/outputs.tf

@@ -0,0 +1,19 @@
+output "apm_url" {
+  value = module.ec_deployment.apm_url
+}
+
+output "es_url" {
+  value = module.ec_deployment.elasticsearch_url
+}
+
+output "username" {
+  value = module.ec_deployment.elasticsearch_username
+}
+output "password" {
+  value     = module.ec_deployment.elasticsearch_password
+  sensitive = true
+}
+
+output "kb_url" {
+  value = module.ec_deployment.kibana_url
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/tags.tf

@@ -0,0 +1,20 @@
+resource "time_static" "created_date" {}
+
+locals {
+  ci_tags = {
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = coalesce(var.CREATED_DATE, time_static.created_date.unix)
+  }
+  project = "apm-server-functionaltest"
+}
+
+module "tags" {
+  source = "../../testing/infra/terraform/modules/tags"
+  # use the convention for team/shared owned resources if we are running in CI. 
+  # assume this is an individually owned resource otherwise. 
+  project = local.project
+}
+

functionaltests/TestUpgrade_8_15_4_to_8_16_0/terraform.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_version = ">= 0.12.29"
+
+  required_providers {
+    ec = {
+      source  = "elastic/ec"
+      version = "0.5.1"
+    }
+  }
+}
+
+locals {
+  api_endpoints = {
+    qa  = "https://public-api.qa.cld.elstc.co"
+    pro = "https://api.elastic-cloud.com"
+  }
+}
+
+provider "ec" {
+  endpoint = local.api_endpoints[var.ec_target]
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/vars.tf

@@ -0,0 +1,47 @@
+variable "ec_target" {
+  type        = string
+  description = "The Elastic Cloud environment to target"
+  validation {
+    condition     = contains(["qa", "pro"], var.ec_target)
+    error_message = "Valid values are (qa, pro)."
+  }
+}
+
+variable "ec_region" {
+  type        = string
+  description = "The Elastic Cloud region to target"
+}
+
+variable "name" {
+  type        = string
+  description = "The deployment name"
+}
+
+variable "stack_version" {
+  type        = string
+  description = "The Elasticsearch version to bootstrap"
+}
+
+# CI variables
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default     = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default     = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default     = ""
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}

functionaltests/go.mod

@@ -0,0 +1,41 @@
+module github.com/elastic/apm-server/functionaltests
+
+go 1.23.2
+
+require (
+	github.com/elastic/apm-perf v0.0.0-20241230130730-2ad47482b731
+	github.com/elastic/go-elasticsearch/v8 v8.16.0
+	github.com/hashicorp/terraform-exec v0.21.0
+	github.com/stretchr/testify v1.10.0
+	go.uber.org/zap v1.27.0
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/hashicorp/terraform-json v0.22.1 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
+	go.elastic.co/apm/v2 v2.6.2 // indirect
+	go.elastic.co/fastjson v1.4.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)