functionaltests: add first test case

functionaltests/8_15_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/apm-server/functionaltests/internal/terraform"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -67,28 +68,28 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 
 	t.Logf("created deployment %s", escfg.KibanaURL)
 
-	ac, err := esclient.New(escfg)
+	ecc, err := esclient.New(escfg)
 	require.NoError(t, err)
 
 	t.Log("creating APM API key")
-	apikey, err := ac.CreateAPMAPIKey(ctx, t.Name())
+	apikey, err := ecc.CreateAPMAPIKey(ctx, t.Name())
 	require.NoError(t, err)
 
 	g := gen.New(escfg.APMServerURL, apikey)
 	g.Logger = zaptest.NewLogger(t, zaptest.Level(zap.InfoLevel))
 
-	previous, err := getDocsCountPerDS(t, ctx, ac)
+	previous, err := getDocsCountPerDS(t, ctx, ecc)
 	require.NoError(t, err)
 
-	g.RunBlockingWait(ctx, ac, expectedIngestForASingleRun(), previous, 1*time.Minute)
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
 
-	beforeUpgradeCount, err := getDocsCountPerDS(t, ctx, ac)
+	beforeUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
 	require.NoError(t, err)
 	assertDocCount(t, beforeUpgradeCount, previous, expectedIngestForASingleRun())
 
 	t.Log("check data streams")
 	var dss []types.DataStream
-	dss, err = ac.GetDataStream(ctx, "*apm*")
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
 	require.NoError(t, err)
 	assertDatastreams(t, checkDatastreamWant{
 		Quantity:         8,
@@ -104,7 +105,7 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 	t.Logf("time elapsed: %s", time.Now().Sub(start))
 
 	t.Log("check number of documents after upgrade")
-	afterUpgradeCount, err := getDocsCountPerDS(t, ctx, ac)
+	afterUpgradeCount, err := getDocsCountPerDS(t, ctx, ecc)
 	require.NoError(t, err)
 	// We assert that no changes happened in the number of documents after upgrade
 	// to ensure the state didn't change before running the next ingestion round
@@ -113,7 +114,7 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 	assertDocCount(t, afterUpgradeCount, esclient.APMDataStreamsDocCount{}, beforeUpgradeCount)
 
 	t.Log("check data streams after upgrade, no rollover expected")
-	dss, err = ac.GetDataStream(ctx, "*apm*")
+	dss, err = ecc.GetDataStream(ctx, "*apm*")
 	require.NoError(t, err)
 	assertDatastreams(t, checkDatastreamWant{
 		Quantity:         8,
@@ -123,18 +124,18 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 		IndicesManagedBy: []string{"Data stream lifecycle"},
 	}, dss)
 
-	g.RunBlockingWait(ctx, ac, expectedIngestForASingleRun(), previous, 1*time.Minute)
+	g.RunBlockingWait(ctx, ecc, expectedIngestForASingleRun(), previous, 1*time.Minute)
 
 	t.Log("check number of documents")
-	afterUpgradeIngestionCount, err := getDocsCountPerDS(t, ctx, ac)
+	afterUpgradeIngestionCount, err := getDocsCountPerDS(t, ctx, ecc)
 	require.NoError(t, err)
 	assertDocCount(t, afterUpgradeIngestionCount, afterUpgradeCount, expectedIngestForASingleRun())
 
 	// Confirm datastreams are
 	// v managed by DSL if created after 8.15.0
 	// x managed by ILM if created before 8.15.0
 	t.Log("check data streams and verify lazy rollover happened")
-	dss2, err := ac.GetDataStream(ctx, "*apm*")
+	dss2, err := ecc.GetDataStream(ctx, "*apm*")
 	require.NoError(t, err)
 	assertDatastreams(t, checkDatastreamWant{
 		Quantity:         8,
@@ -145,6 +146,7 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 	}, dss2)
 	t.Logf("time elapsed: %s", time.Now().Sub(start))
 
-	// check ES logs, there should be no errors
-	// TODO: how to get these from Elastic Cloud? Is it possible?
+	res, err := ecc.GetESErrorLogs(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, res.Hits.Total.Value)
 }

functionaltests/TestUpgrade_8_15_4_to_8_16_0/main.tf

@@ -5,6 +5,10 @@ module "ec_deployment" {
   deployment_template    = "aws-storage-optimized"
   deployment_name_prefix = var.name
 
+  // self monitoring is enabled so we can inspect Elasticsearch
+  // logs from tests.
+  observability_deployment = "self"
+
   apm_server_size = "1g"
 
   elasticsearch_size       = "4g"

functionaltests/internal/esclient/client.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/esql/query"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/security/createapikey"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
@@ -157,3 +158,36 @@ line 1:1: Unknown index [traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-
 
 	return res, nil
 }
+
+// GetESErrorLogs retrieves Elasticsearch error logs.
+// The search query is on the Index used by Elasticsearch monitoring to store logs.
+func (c *Client) GetESErrorLogs(ctx context.Context) (*search.Response, error) {
+	res, err := c.es.Search().
+		Index("elastic-cloud-logs-8").
+		Request(&search.Request{
+			Query: &types.Query{
+				Bool: &types.BoolQuery{
+					Must: []types.Query{
+						{
+							Match: map[string]types.MatchQuery{
+								"service.type": {Query: "elasticsearch"},
+							},
+						},
+						{
+							Match: map[string]types.MatchQuery{
+								"log.level": {Query: "ERROR"},
+							},
+						},
+					},
+				},
+			},
+		}).Do(ctx)
+	if err != nil {
+		return search.NewResponse(), fmt.Errorf("cannot run search query: %w", err)
+	}
+
+	fmt.Printf("%+v\n", res)
+	fmt.Println(res.Hits.Total.Value)
+
+	return res, nil
+}

functionaltests/main_test.go

@@ -36,10 +36,6 @@ var cleanupOnFailure *bool = flag.Bool("cleanup-on-failure", true, "Whether to r
 // We use 'pro' for production as that is the key used to retrieve EC_API_KEY from secret storage.
 var target *string = flag.String("target", "pro", "The target environment where to run tests againts. Valid values are: qa, pro")
 
-// https://www.elastic.co/guide/en/cloud/current/ec-regions-templates-instances.html
-const testRegionQA = "aws-eu-west-1"
-const testRegionProduction = "eu-west-1"
-
 // expectedIngestForASingleRun() represent the expected number of ingested document after a
 // single run of ingest().
 // Only non aggregation data streams are included, as aggregation ones differs on different
@@ -53,10 +49,6 @@ func expectedIngestForASingleRun() esclient.APMDataStreamsDocCount {
 	}
 }
 
-// docsPerDatastream is a utility type to map esclient.ApmDocCount to a format
-// easier to perform assertions on.
-type docsPerDatastream map[string]int
-
 // getDocsCountPerDS retrieves document count.
 func getDocsCountPerDS(t *testing.T, ctx context.Context, ecc *esclient.Client) (esclient.APMDataStreamsDocCount, error) {
 	t.Helper()
@@ -113,12 +105,13 @@ func assertDatastreams(t *testing.T, expected checkDatastreamWant, actual []type
 
 // regionFrom returns the appropriate region to run test
 // againts based on specified target.
+// https://www.elastic.co/guide/en/cloud/current/ec-regions-templates-instances.html
 func regionFrom(target string) string {
 	switch target {
 	case "qa":
-		return testRegionQA
+		return "aws-eu-west-1"
 	case "pro":
-		return testRegionProduction
+		return "eu-west-1"
 	default:
 		panic("target value is not accepted")
 	}