functionaltests: filter unwanted logs (#15497)

We observed error logs unrelated to APM Server. We don't wont such
logs to fail these tests, so this commit includes filtering logic
based on the log message content.

functionaltests/8_15_test.go

@@ -25,13 +25,13 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
+	"github.com/elastic/apm-server/functionaltests/internal/asserts"
 	"github.com/elastic/apm-server/functionaltests/internal/ecclient"
 	"github.com/elastic/apm-server/functionaltests/internal/esclient"
 	"github.com/elastic/apm-server/functionaltests/internal/gen"
 	"github.com/elastic/apm-server/functionaltests/internal/terraform"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -154,12 +154,7 @@ func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
 	}, dss2)
 	t.Logf("time elapsed: %s", time.Now().Sub(start))
 
-	res, err := ecc.GetESErrorLogs(ctx)
+	resp, err := ecc.GetESErrorLogs(ctx)
 	require.NoError(t, err)
-	if !assert.Zero(t, res.Hits.Total.Value, "expected no error logs, but found some") {
-		t.Log("found error logs:")
-		for _, h := range res.Hits.Hits {
-			t.Log(string(h.Source_))
-		}
-	}
+	asserts.ZeroESLogs(t, *resp)
 }

functionaltests/internal/asserts/doc.go

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package asserts provides assertion helpers for our functional tests.
+// Assertions can be implemented as public functions, hiding the additional
+// logic/types required for a more ergonomic implementation and with
+// unit tests coverage to ease development.
+package asserts

functionaltests/internal/asserts/es_logs.go

@@ -0,0 +1,95 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package asserts
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// loggerESlog is used to unmarshal RawJSON from search responses
+// to allow filtering on specific fields.
+type loggerESlog struct {
+	Message       string
+	Elasticsearch struct {
+		Server struct {
+			ErrorMessage string `json:"error.message"`
+		}
+	}
+	Log struct {
+		Logger string
+	}
+}
+
+// includeLog returns a boolean indicating if the specified log
+// should be included in further processing.
+func includeLog(l loggerESlog) bool {
+	switch l.Log.Logger {
+	case "org.elasticsearch.ingest.geoip.GeoIpDownloader":
+		return false
+	}
+
+	return true
+}
+
+func ZeroESLogs(t *testing.T, resp search.Response) {
+	t.Helper()
+
+	// Total is present only if `track_total_hits` wasn't `false` in
+	// the search request. Guard against an unexpected panic.
+	// This may also happen if the fixture does not include it.
+	if resp.Hits.Total == nil {
+		panic("hits.total.value is missing, are you setting track_total_hits=false in the request or is it missing from the fixture?")
+	}
+
+	// if there are no error logs, we are done here.
+	if resp.Hits.Total.Value == 0 {
+		return
+	}
+
+	// otherwise apply filters an assert what's left
+	logs := []loggerESlog{}
+	for _, v := range resp.Hits.Hits {
+		var l loggerESlog
+		require.NoError(t, json.Unmarshal(v.Source_, &l))
+
+		if includeLog(l) {
+			logs = append(logs, l)
+		}
+	}
+
+	if !assert.Len(t, logs, 0, "expected no error logs, but found some") {
+		// As most of the errors we face here are not related to APM Server
+		// and are not consistently appearing, we need a way to easily extract
+		// the response for fitlering it out. Response is then used in tests.
+		t.Log("printing response for adding test cases:")
+		r, err := json.Marshal(resp)
+		require.NoError(t, err, "cannot marshal response")
+		t.Log(string(r))
+
+		t.Log("found error logs:")
+		for _, l := range logs {
+			t.Logf("- (%s) %s: %s", l.Log.Logger, l.Message, l.Elasticsearch.Server.ErrorMessage)
+		}
+	}
+}

functionaltests/internal/asserts/es_logs_test.go

@@ -0,0 +1,57 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package asserts_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/elastic/apm-server/functionaltests/internal/asserts"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestZeroESLogs(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+	}{
+		{
+			name:     "with no logs",
+			response: `{"hits":{"hits":[], "total": {"value": 0}}}`,
+		},
+		{
+			name: "ignore org.elasticsearch.ingest.geoip.GeoIpDownloader",
+			// this is not an entire response because it has been retrived before we added the logic to display it.
+			response: `{"hits":{"hits":[{"_index":"","_source":{"agent":{"name":"df6e53b971fd","id":"7441fe9e-6f94-483b-881c-e218feeb8997","ephemeral_id":"8b18f265-3238-44c3-ac1a-2d51c31c532e","type":"filebeat","version":"8.16.0"},"process":{"thread":{"name":"elasticsearch[instance-0000000000][generic][T#14]"}},"log":{"file":{"path":"/app/logs/dcf3a6e1e25e4537a084c846786925c7_server.json"},"offset":713250,"level":"ERROR","logger":"org.elasticsearch.ingest.geoip.GeoIpDownloader"},"fileset":{"name":"server"},"message":"exception during geoip databases update","cloud":{"availability_zone":"eu-west-1b"},"input":{"type":"log"},"@timestamp":"2025-01-27T09:01:51.286Z","ecs":{"version":"1.2.0"},"elasticsearch":{"server":{"process":{"thread":{}},"log":{},"error.type":"org.elasticsearch.ElasticsearchException","ecs":{},"elasticsearch":{"cluster":{},"node":{}},"error.message":"not all primary shards of [.geoip_databases] index are active","service":{},"error.stack_trace":"org.elasticsearch.ElasticsearchException: not all primary shards of [.geoip_databases] index are active\n\tat [email protected]/org.elasticsearch.ingest.geoip.GeoIpDownloader.updateDatabases(GeoIpDownloader.java:142)\n\tat [email protected]/org.elasticsearch.ingest.geoip.GeoIpDownloader.runDownloader(GeoIpDownloader.java:294)\n\tat [email protected]/org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor.nodeOperation(GeoIpDownloaderTaskExecutor.java:165)\n\tat [email protected]/org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor.nodeOperation(GeoIpDownloaderTaskExecutor.java:64)\n\tat [email protected]/org.elasticsearch.persistent.NodePersistentTasksExecutor$1.doRun(NodePersistentTasksExecutor.java:35)\n\tat [email protected]/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1023)\n\tat [email protected]/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1575)\n","event":{}},"cluster":{"name":"dcf3a6e1e25e4537a084c846786925c7","uuid":"M0sVrLQCRiWaVIkY6R4RpA"},"node":{"name":"instance-0000000000","id":"mb8RZp6TS6KhNboqoOd7xA"}},"service":{"node":{"name":"instance-0000000000"},"name":"ES_ECS","id":"edfcb0613ad26188c3a11c977a40970b","type":"elasticsearch","version":"8.16.0"},"host":{"name":"instance-0000000000","id":"mb8RZp6TS6KhNboqoOd7xA"},"event":{"ingested":"2025-01-27T09:03:51.502916073Z","created":"2025-01-27T09:01:57.494Z","kind":"event","module":"elasticsearch","category":"database","type":"error","dataset":"elasticsearch.server"},"deployment":{"name":"TestUpgrade_8_15_4_to_8_16_0-8.15.4"}}}],"total":{"relation":"","value":1}},"_shards":{"failed":0,"successful":0,"total":0},"timed_out":false,"took":0}`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var resp search.Response
+			require.NoError(t, json.Unmarshal([]byte(tt.response), &resp))
+
+			asserts.ZeroESLogs(t, resp)
+			// we only check that we running ZeroESLogs did not fail this test.
+			assert.False(t, t.Failed())
+		})
+	}
+}