[8.19] (backport #11372) (chore): remove duplicate integration tests (#11467)

* (chore): remove duplicate integration tests (#11372)

* (chore): remove duplicate integration tests

* add hybrid agent test back

* fix test

(cherry picked from commit 4664a08)

# Conflicts:
#	testing/integration/ess/otel_test.go

* fix conflict

---------

Co-authored-by: Khushi Jain <[email protected]>

Author: mergify[bot]

testing/integration/ess/otel_test.go

@@ -794,263 +794,6 @@ func mapAtLeastOneTrue(mm map[string]bool) bool {
 	return false
 }
 
-func TestFileBeatReceiver(t *testing.T) {
-	define.Require(t, define.Requirements{
-		Group: integration.Default,
-		Local: true,
-		OS: []define.OS{
-			{Type: define.Windows},
-			{Type: define.Linux},
-			{Type: define.Darwin},
-		},
-	})
-
-	type otelConfigOptions struct {
-		Message string
-		Output  string
-		HomeDir string
-	}
-	testMessage := "supercalifragilisticexpialidocious"
-	tmpDir := t.TempDir()
-	exporterOutputPath := filepath.Join(tmpDir, "output.json")
-	t.Cleanup(func() {
-		if t.Failed() {
-			contents, err := os.ReadFile(exporterOutputPath)
-			if err != nil {
-				t.Logf("No exporter output file")
-				return
-			}
-			t.Logf("Contents of exporter output file:\n%s\n", string(contents))
-		}
-	})
-	otelConfigPath := filepath.Join(tmpDir, "otel.yml")
-	otelConfigTemplate := `receivers:
-  filebeatreceiver:
-    filebeat:
-      inputs:
-        - type: benchmark
-          enabled: true
-          count: 1
-          message: {{.Message}}
-    output:
-      otelconsumer:
-    logging:
-      level: info
-      selectors:
-        - '*'
-    path.home: {{.HomeDir}}
-exporters:
-  file/no_rotation:
-    path: {{.Output}}
-service:
-  pipelines:
-    logs:
-      receivers: [filebeatreceiver]
-      exporters: [file/no_rotation]
-`
-
-	var otelConfigBuffer bytes.Buffer
-	require.NoError(t,
-		template.Must(template.New("otelConfig").Parse(otelConfigTemplate)).Execute(&otelConfigBuffer,
-			otelConfigOptions{
-				Message: testMessage,
-				Output:  exporterOutputPath,
-				HomeDir: tmpDir,
-			}))
-	require.NoError(t, os.WriteFile(otelConfigPath, otelConfigBuffer.Bytes(), 0o600))
-	t.Cleanup(func() {
-		if t.Failed() {
-			contents, err := os.ReadFile(otelConfigPath)
-			if err != nil {
-				t.Logf("no otel config file")
-				return
-			}
-			t.Logf("Contents of otel config file:\n%s\n", string(contents))
-		}
-	})
-	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), aTesting.WithAdditionalArgs([]string{"--config", otelConfigPath}))
-	require.NoError(t, err)
-
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(5*time.Minute))
-	defer cancel()
-	err = fixture.Prepare(ctx, fakeComponent)
-	require.NoError(t, err)
-
-	var fixtureWg sync.WaitGroup
-	fixtureWg.Add(1)
-	go func() {
-		defer fixtureWg.Done()
-		err = fixture.RunOtelWithClient(ctx)
-	}()
-
-	require.Eventually(t,
-		func() bool {
-			content, err := os.ReadFile(exporterOutputPath)
-			if err != nil || len(content) == 0 {
-				return false
-			}
-			return bytes.Contains(content, []byte(testMessage))
-		},
-		3*time.Minute, 1*time.Second,
-		fmt.Sprintf("there should be exported logs by now"))
-
-	cancel()
-	fixtureWg.Wait()
-	require.True(t, err == nil || err == context.Canceled || err == context.DeadlineExceeded, "Retrieved unexpected error: %s", err.Error())
-}
-
-func TestOtelFBReceiverE2E(t *testing.T) {
-	info := define.Require(t, define.Requirements{
-		Group: integration.Default,
-		Local: true,
-		OS: []define.OS{
-			{Type: define.Windows},
-			{Type: define.Linux},
-			{Type: define.Darwin},
-		},
-		Stack: &define.Stack{},
-	})
-	tmpDir := t.TempDir()
-	numEvents := 50
-	// Create the data file to ingest
-	inputFile, err := os.CreateTemp(tmpDir, "input.txt")
-	require.NoError(t, err, "failed to create temp file to hold data to ingest")
-	inputFilePath := inputFile.Name()
-	for i := 0; i < numEvents; i++ {
-		_, err = inputFile.Write([]byte(fmt.Sprintf("Line %d\n", i)))
-		require.NoErrorf(t, err, "failed to write line %d to temp file", i)
-	}
-	err = inputFile.Close()
-	require.NoError(t, err, "failed to close data temp file")
-	t.Cleanup(func() {
-		if t.Failed() {
-			contents, err := os.ReadFile(inputFilePath)
-			if err != nil {
-				t.Logf("no data file to import at %s", inputFilePath)
-				return
-			}
-			t.Logf("contents of import file:\n%s\n", string(contents))
-		}
-	})
-
-	// Create the otel configuration file
-	type otelConfigOptions struct {
-		InputPath  string
-		HomeDir    string
-		ESEndpoint string
-		ESApiKey   string
-		Index      string
-		MinItems   int
-	}
-	esEndpoint, err := integration.GetESHost()
-	require.NoError(t, err, "error getting elasticsearch endpoint")
-	esApiKey, err := createESApiKey(info.ESClient)
-	require.NoError(t, err, "error creating API key")
-	require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey)
-	index := "logs-integration-default"
-	otelConfigTemplate := `receivers:
-  filebeatreceiver:
-    filebeat:
-      inputs:
-        - type: filestream
-          id: filestream-end-to-end
-          enabled: true
-          paths:
-            - {{.InputPath}}
-          prospector.scanner.fingerprint.enabled: false
-          file_identity.native: ~
-    output:
-      otelconsumer:
-    logging:
-      level: info
-      selectors:
-        - '*'
-    path.home: {{.HomeDir}}
-    queue.mem.flush.timeout: 0s
-exporters:
-  elasticsearch/log:
-    endpoints:
-      - {{.ESEndpoint}}
-    api_key: {{.ESApiKey}}
-    logs_index: {{.Index}}
-    sending_queue:
-      wait_for_result: true # Avoid losing data on shutdown
-      block_on_overflow: true
-      batch:
-        flush_timeout: 1s
-        min_size: {{.MinItems}}
-    mapping:
-      mode: bodymap
-service:
-  pipelines:
-    logs:
-      receivers:
-        - filebeatreceiver
-      exporters:
-        - elasticsearch/log
-`
-	otelConfigPath := filepath.Join(tmpDir, "otel.yml")
-	var otelConfigBuffer bytes.Buffer
-	require.NoError(t,
-		template.Must(template.New("otelConfig").Parse(otelConfigTemplate)).Execute(&otelConfigBuffer,
-			otelConfigOptions{
-				InputPath:  inputFilePath,
-				HomeDir:    tmpDir,
-				ESEndpoint: esEndpoint,
-				ESApiKey:   esApiKey.Encoded,
-				Index:      index,
-				MinItems:   numEvents,
-			}))
-	require.NoError(t, os.WriteFile(otelConfigPath, otelConfigBuffer.Bytes(), 0o600))
-	t.Cleanup(func() {
-		if t.Failed() {
-			contents, err := os.ReadFile(otelConfigPath)
-			if err != nil {
-				t.Logf("No otel configuration file at %s", otelConfigPath)
-				return
-			}
-			t.Logf("Contents of otel config file:\n%s\n", string(contents))
-		}
-	})
-	// Now we can actually create the fixture and run it
-	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), aTesting.WithAdditionalArgs([]string{"--config", otelConfigPath}))
-	require.NoError(t, err)
-
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(5*time.Minute))
-	defer cancel()
-	err = fixture.Prepare(ctx, fakeComponent)
-	require.NoError(t, err)
-
-	var fixtureWg sync.WaitGroup
-	fixtureWg.Add(1)
-	go func() {
-		defer fixtureWg.Done()
-		err = fixture.RunOtelWithClient(ctx)
-	}()
-
-	// Make sure find the logs
-	actualHits := &struct{ Hits int }{}
-	require.Eventually(t,
-		func() bool {
-			findCtx, findCancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer findCancel()
-
-			docs, err := estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+index+"*", map[string]interface{}{
-				"log.file.path": inputFilePath,
-			})
-			require.NoError(t, err)
-
-			actualHits.Hits = docs.Hits.Total.Value
-			return actualHits.Hits == numEvents
-		},
-		2*time.Minute, 1*time.Second,
-		"Expected %d logs, got %v", numEvents, actualHits)
-
-	cancel()
-	fixtureWg.Wait()
-	require.True(t, err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded), "Retrieved unexpected error: %s", err.Error())
-}
-
 func TestOtelFilestreamInput(t *testing.T) {
 	info := define.Require(t, define.Requirements{
 		Group: integration.Default,
@@ -1346,135 +1089,6 @@ agent.monitoring:
 	cmd.Wait()
 }
 
-func TestOtelMBReceiverE2E(t *testing.T) {
-	info := define.Require(t, define.Requirements{
-		Group: integration.Default,
-		Local: true,
-		OS: []define.OS{
-			// {Type: define.Windows}, we don't support otel on Windows yet
-			{Type: define.Linux},
-			{Type: define.Darwin},
-		},
-		Stack: &define.Stack{},
-	})
-	tmpDir := t.TempDir()
-
-	// Create the otel configuration file
-	type otelConfigOptions struct {
-		HomeDir    string
-		ESEndpoint string
-		ESApiKey   string
-		Index      string
-		MinItems   int
-	}
-	esEndpoint, err := integration.GetESHost()
-	require.NoError(t, err, "error getting elasticsearch endpoint")
-	esApiKey, err := createESApiKey(info.ESClient)
-	require.NoError(t, err, "error creating API key")
-	require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey)
-	index := "logs-integration-default"
-	otelConfigTemplate := `receivers:
-  metricbeatreceiver:
-    metricbeat:
-      modules:
-        - module: system
-          enabled: true
-          period: 1s
-          processes:
-            - '.*'
-          metricsets:
-            - cpu
-    output:
-      otelconsumer:
-    logging:
-      level: info
-      selectors:
-        - '*'
-    path.home: {{.HomeDir}}
-    queue.mem.flush.timeout: 0s
-exporters:
-  elasticsearch/log:
-    endpoints:
-      - {{.ESEndpoint}}
-    api_key: {{.ESApiKey}}
-    logs_index: {{.Index}}
-    sending_queue:
-      wait_for_result: true # Avoid losing data on shutdown
-      block_on_overflow: true
-      batch:
-        flush_timeout: 1s
-        min_size: {{.MinItems}}
-    mapping:
-      mode: bodymap
-service:
-  pipelines:
-    logs:
-      receivers:
-        - metricbeatreceiver
-      exporters:
-        - elasticsearch/log
-`
-	otelConfigPath := filepath.Join(tmpDir, "otel.yml")
-	var otelConfigBuffer bytes.Buffer
-	require.NoError(t,
-		template.Must(template.New("otelConfig").Parse(otelConfigTemplate)).Execute(&otelConfigBuffer,
-			otelConfigOptions{
-				HomeDir:    tmpDir,
-				ESEndpoint: esEndpoint,
-				ESApiKey:   esApiKey.Encoded,
-				Index:      index,
-				MinItems:   1,
-			}))
-	require.NoError(t, os.WriteFile(otelConfigPath, otelConfigBuffer.Bytes(), 0o600))
-	t.Cleanup(func() {
-		if t.Failed() {
-			contents, err := os.ReadFile(otelConfigPath)
-			if err != nil {
-				t.Logf("No otel configuration file at %s", otelConfigPath)
-				return
-			}
-			t.Logf("Contents of otel config file:\n%s\n", string(contents))
-		}
-	})
-	// Now we can actually create the fixture and run it
-	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), aTesting.WithAdditionalArgs([]string{"--config", otelConfigPath}))
-	require.NoError(t, err)
-
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(5*time.Minute))
-	defer cancel()
-	err = fixture.Prepare(ctx, fakeComponent)
-	require.NoError(t, err)
-
-	var fixtureWg sync.WaitGroup
-	fixtureWg.Add(1)
-	go func() {
-		defer fixtureWg.Done()
-		err = fixture.RunOtelWithClient(ctx)
-	}()
-
-	// Make sure find the logs
-	actualHits := &struct{ Hits int }{}
-	require.Eventually(t,
-		func() bool {
-			findCtx, findCancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer findCancel()
-
-			docs, err := estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+index+"*", map[string]interface{}{
-				"metricset.name": "cpu",
-			})
-			require.NoError(t, err)
-
-			actualHits.Hits = docs.Hits.Total.Value
-			return actualHits.Hits >= 1
-		},
-		2*time.Minute, 1*time.Second,
-		"Expected at least %d logs, got %v", 1, actualHits)
-
-	cancel()
-	fixtureWg.Wait()
-	require.True(t, err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded), "Retrieved unexpected error: %s", err.Error())
-}
-
 func TestHybridAgentE2E(t *testing.T) {
 	// This test is a hybrid agent test that ingests a single log with
 	// filebeat and fbreceiver. It then compares the final documents in