[9.1] (backport #10052) [otel] Implement EDOT diagnostics extension

internal/pkg/agent/application/paths/common.go

@@ -33,6 +33,9 @@ const (
 	// ControlSocketName is the control socket name.
 	ControlSocketName = "elastic-agent.sock"
 
+	// Our DiagnosticsExtension will use DiagnosticsExtensionSocketName to listen and serve diagnostic requests.
+	DiagnosticsExtensionSocketName = "edot-diagnostics-extension.sock"
+
 	// WindowsControlSocketInstalledPath is the control socket path used when installed on Windows.
 	WindowsControlSocketInstalledPath = `npipe:///elastic-agent-system`
 
@@ -52,16 +55,17 @@ const (
 var ExternalInputsPattern = filepath.Join("inputs.d", "*.yml")
 
 var (
-	topPath           string
-	configPath        string
-	configFilePath    string
-	logsPath          string
-	downloadsPath     string
-	componentsPath    string
-	installPath       string
-	controlSocketPath string
-	unversionedHome   bool
-	tmpCreator        sync.Once
+	topPath                    string
+	configPath                 string
+	configFilePath             string
+	logsPath                   string
+	downloadsPath              string
+	componentsPath             string
+	installPath                string
+	controlSocketPath          string
+	diagnosticsExtensionSocket string
+	unversionedHome            bool
+	tmpCreator                 sync.Once
 )
 
 func init() {
@@ -70,6 +74,7 @@ func init() {
 	configPath = topPath
 	logsPath = topPath
 	controlSocketPath = initialControlSocketPath(topPath)
+	diagnosticsExtensionSocket = SocketFromPath(runtime.GOOS, topPath, DiagnosticsExtensionSocketName)
 	unversionedHome = false // only versioned by container subcommand
 
 	// these should never change
@@ -339,8 +344,14 @@ func RunningInstalled() bool {
 // ControlSocketFromPath returns the control socket path for an Elastic Agent running
 // on the defined platform, and its executing directory.
 func ControlSocketFromPath(platform string, path string) string {
+	return SocketFromPath(platform, path, ControlSocketName)
+}
+
+// SocketFromPath returns the socket path for an Elastic Agent running
+// on the defined platform for a given socket, and its executing directory.
+func SocketFromPath(platform string, path string, socketName string) string {
 	// socket should be inside this directory
-	socketPath := filepath.Join(path, ControlSocketName)
+	socketPath := filepath.Join(path, socketName)
 	if platform == "windows" {
 		// on windows the control socket always uses the fallback
 		return utils.SocketURLWithFallback(socketPath, path)
@@ -355,6 +366,14 @@ func ControlSocketFromPath(platform string, path string) string {
 	return utils.SocketURLWithFallback(socketPath, path)
 }
 
+func DiagnosticsExtensionSocket() string {
+	return diagnosticsExtensionSocket
+}
+
+func SetDiagnosticsExtensionSocket(socket string) {
+	diagnosticsExtensionSocket = SocketFromPath(runtime.GOOS, topPath, socket)
+}
+
 func pathSplit(path string) []string {
 	dir, file := filepath.Split(path)
 	if dir == "" && file == "" {

internal/pkg/agent/cmd/diagnostics_otel.go

@@ -0,0 +1,113 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
+	"github.com/elastic/elastic-agent/internal/pkg/cli"
+	"github.com/elastic/elastic-agent/internal/pkg/diagnostics"
+	"github.com/elastic/elastic-agent/internal/pkg/otel"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
+)
+
+func newOtelDiagnosticsCommand(streams *cli.IOStreams) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "diagnostics",
+		Short: "Gather diagnostics information from the EDOT and write it to a zip archive",
+		Long:  "This command gathers diagnostics information from the EDOT and writes it to a zip archive",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if err := otelDiagnosticCmd(streams, cmd); err != nil {
+				fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage())
+				os.Exit(1)
+			}
+			return nil
+		},
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+	cmd.Flags().StringP("file", "f", "", "name of the output diagnostics zip archive")
+	cmd.Flags().BoolP("cpu-profile", "p", false, "wait to collect a CPU profile")
+	return cmd
+}
+
+func otelDiagnosticCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
+	cpuProfile, _ := cmd.Flags().GetBool("cpu-profile")
+	resp, err := otel.PerformDiagnosticsExt(cmd.Context(), cpuProfile)
+	if err != nil {
+		return fmt.Errorf("failed to get edot diagnostics: %w", err)
+	}
+
+	agentDiag := make([]client.DiagnosticFileResult, 0)
+	for _, r := range resp.GlobalDiagnostics {
+		agentDiag = append(agentDiag, client.DiagnosticFileResult{
+			Name:        r.Name,
+			Filename:    r.Filename,
+			ContentType: r.ContentType,
+			Content:     r.Content,
+			Description: r.Description,
+		})
+	}
+
+	componentDiag := make([]client.DiagnosticComponentResult, 0)
+	for _, r := range resp.ComponentDiagnostics {
+		res := client.DiagnosticComponentResult{
+			Results: make([]client.DiagnosticFileResult, 0),
+		}
+		res.Results = append(res.Results, client.DiagnosticFileResult{
+			Name:        r.Name,
+			Filename:    r.Filename,
+			ContentType: r.ContentType,
+			Content:     r.Content,
+			Description: r.Description,
+		})
+		res.ComponentID = r.Name
+		componentDiag = append(componentDiag, res)
+	}
+	componentDiag = aggregateComponentDiagnostics(componentDiag)
+
+	filepath, _ := cmd.Flags().GetString("file")
+	if filepath == "" {
+		ts := time.Now().UTC()
+		filepath = "edot-diagnostics-" + ts.Format("2006-01-02T15-04-05Z07-00") + ".zip" // RFC3339 format that replaces : with -, so it will work on Windows
+	}
+	f, err := createFile(filepath)
+	if err != nil {
+		return fmt.Errorf("could not create diagnostics file %q: %w", filepath, err)
+	}
+	defer f.Close()
+
+	// In EDOT, the logs path does not exist, so we ignore that error.
+	if err := diagnostics.ZipArchive(streams.Err, f, paths.Top(), agentDiag, nil, componentDiag, false); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to create archive %q: %w", filepath, err)
+	}
+	fmt.Fprintf(streams.Out, "Created diagnostics archive %q\n", filepath)
+	fmt.Fprintln(streams.Out, "** WARNING **\nCreated archive may contain plain text credentials.\nEnsure that files in archive are redacted before sharing.\n*******")
+	return nil
+}
+
+// aggregateComponentDiagnostics takes a slice of DiagnosticComponentResult and merges
+// results for components with the same ComponentID.
+func aggregateComponentDiagnostics(diags []client.DiagnosticComponentResult) []client.DiagnosticComponentResult {
+	m := make(map[string]client.DiagnosticComponentResult)
+	for _, d := range diags {
+		if existing, ok := m[d.ComponentID]; ok {
+			existing.Results = append(existing.Results, d.Results...)
+			m[d.ComponentID] = existing
+		} else {
+			m[d.ComponentID] = d
+		}
+	}
+	result := make([]client.DiagnosticComponentResult, 0, len(m))
+	for _, v := range m {
+		result = append(result, v)
+	}
+	return result
+}

internal/pkg/agent/cmd/otel.go

@@ -47,6 +47,7 @@ func newOtelCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Comman
 	setupOtelFlags(cmd.Flags())
 	cmd.AddCommand(newValidateCommandWithArgs(args, streams))
 	cmd.AddCommand(newComponentsCommandWithArgs(args, streams))
+	cmd.AddCommand(newOtelDiagnosticsCommand(streams))
 
 	return cmd
 }

internal/pkg/agent/cmd/run.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/elastic-agent-libs/service"
 	"github.com/elastic/elastic-agent-system-metrics/report"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/vault"
+	"github.com/elastic/elastic-agent/internal/pkg/diagnostics"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
@@ -50,7 +51,6 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
 	monitoringCfg "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config"
-	"github.com/elastic/elastic-agent/internal/pkg/diagnostics"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/control/v2/server"

internal/pkg/diagnostics/diagnostics.go

@@ -226,8 +226,9 @@ func ZipArchive(
 	if err != nil {
 		return fmt.Errorf("error creating .zip header for components/ directory: %w", err)
 	}
+
 	// iterate over components
-	for dirName, units := range compDirs {
+	for dirName, comp := range componentResults {
 		_, err := zw.CreateHeader(&zip.FileHeader{
 			Name:     fmt.Sprintf("components/%s/", dirName),
 			Method:   zip.Deflate,
@@ -236,66 +237,65 @@ func ZipArchive(
 		if err != nil {
 			return fmt.Errorf("error creating .zip header for component directory: %w", err)
 		}
-		// create component diags
-		if comp, ok := componentResults[dirName]; ok {
-			// check for component-level errors
-			if comp.Err != nil {
-				err = writeErrorResult(zw, fmt.Sprintf("components/%s/error.txt", dirName), comp.Err.Error())
+		if comp.Err != nil {
+			err = writeErrorResult(zw, fmt.Sprintf("components/%s/error.txt", dirName), comp.Err.Error())
+			if err != nil {
+				return fmt.Errorf("error while writing error result for component %s: %w", comp.ComponentID, err)
+			}
+		} else {
+			for _, res := range comp.Results {
+
+				filePath := fmt.Sprintf("components/%s/%s", dirName, res.Filename)
+				resFileWriter, err := zw.CreateHeader(&zip.FileHeader{
+					Name:     filePath,
+					Method:   zip.Deflate,
+					Modified: ts,
+				})
 				if err != nil {
-					return fmt.Errorf("error while writing error result for component %s: %w", comp.ComponentID, err)
+					return fmt.Errorf("error creating .zip header for %s: %w", res.Filename, err)
 				}
-			} else {
-				for _, res := range comp.Results {
-
-					filePath := fmt.Sprintf("components/%s/%s", dirName, res.Filename)
-					resFileWriter, err := zw.CreateHeader(&zip.FileHeader{
-						Name:     filePath,
-						Method:   zip.Deflate,
-						Modified: ts,
-					})
-					if err != nil {
-						return fmt.Errorf("error creating .zip header for %s: %w", res.Filename, err)
-					}
-					err = writeRedacted(errOut, resFileWriter, filePath, res)
-					if err != nil {
-						return fmt.Errorf("error writing %s in zip file: %w", res.Filename, err)
-					}
+				err = writeRedacted(errOut, resFileWriter, filePath, res)
+				if err != nil {
+					return fmt.Errorf("error writing %s in zip file: %w", res.Filename, err)
 				}
 			}
-
 		}
 		// create unit diags
-		for _, ud := range units {
-			unitDir := strings.ReplaceAll(strings.TrimPrefix(ud.UnitID, ud.ComponentID+"-"), "/", "-")
-			_, err := zw.CreateHeader(&zip.FileHeader{
-				Name:     fmt.Sprintf("components/%s/%s/", dirName, unitDir),
-				Method:   zip.Deflate,
-				Modified: ts,
-			})
-			if err != nil {
-				return fmt.Errorf("error creating .zip header for unit directory: %w", err)
-			}
-			// check for unit-level errors
-			if ud.Err != nil {
-				err = writeErrorResult(zw, fmt.Sprintf("components/%s/%s/error.txt", dirName, unitDir), ud.Err.Error())
-				if err != nil {
-					return fmt.Errorf("error while writing error result for unit %s: %w", ud.UnitID, err)
-				}
-				continue
-			}
-			for _, fr := range ud.Results {
-				filePath := fmt.Sprintf("components/%s/%s/%s", dirName, unitDir, fr.Filename)
-				w, err := zw.CreateHeader(&zip.FileHeader{
-					Name:     filePath,
+		if units, ok := compDirs[dirName]; ok {
+			// check for component-level errors
+			// create unit diags
+			for _, ud := range units {
+				unitDir := strings.ReplaceAll(strings.TrimPrefix(ud.UnitID, ud.ComponentID+"-"), "/", "-")
+				_, err := zw.CreateHeader(&zip.FileHeader{
+					Name:     fmt.Sprintf("components/%s/%s/", dirName, unitDir),
 					Method:   zip.Deflate,
-					Modified: fr.Generated,
+					Modified: ts,
 				})
 				if err != nil {
-					return err
+					return fmt.Errorf("error creating .zip header for unit directory: %w", err)
 				}
-				err = writeRedacted(errOut, w, filePath, fr)
-				if err != nil {
-					return err
+				// check for unit-level errors
+				if ud.Err != nil {
+					err = writeErrorResult(zw, fmt.Sprintf("components/%s/%s/error.txt", dirName, unitDir), ud.Err.Error())
+					if err != nil {
+						return fmt.Errorf("error while writing error result for unit %s: %w", ud.UnitID, err)
+					}
+					continue
+				}
+				for _, fr := range ud.Results {
+					filePath := fmt.Sprintf("components/%s/%s/%s", dirName, unitDir, fr.Filename)
+					w, err := zw.CreateHeader(&zip.FileHeader{
+						Name:     filePath,
+						Method:   zip.Deflate,
+						Modified: fr.Generated,
+					})
+					if err != nil {
+						return err
+					}
+					err = writeRedacted(errOut, w, filePath, fr)
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}

internal/pkg/otel/components.go

@@ -67,6 +67,8 @@ import (
 	filestorage "github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage"
 	"go.opentelemetry.io/collector/extension/memorylimiterextension" // for putting backpressure when approach a memory limit
 
+	elasticdiagnostics "github.com/elastic/elastic-agent/internal/pkg/otel/extension/elasticdiagnostics"
+
 	"github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension"
 	"github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension"
 
@@ -171,6 +173,7 @@ func components(extensionFactories ...extension.Factory) func() (otelcol.Factori
 			apmconfigextension.NewFactory(),
 			beatsauthextension.NewFactory(),
 			headersetterextension.NewFactory(),
+			elasticdiagnostics.NewFactory(),
 		}
 		extensions = append(extensions, extensionFactories...)
 		factories.Extensions, err = otelcol.MakeFactoryMap[extension.Factory](extensions...)