Skip to content

Commit 1b7a5e8

Browse files
swiatekmmergify[bot]
swiatekm
authored and
mergify[bot]
committed
Deduplicate extensions when generating otel config (#10172)
(cherry picked from commit 5aacbcb) # Conflicts: # internal/pkg/otel/translate/otelconfig.go # internal/pkg/otel/translate/otelconfig_test.go
1 parent d8ea4a6 commit 1b7a5e8

File tree

2 files changed

+217
-6
lines changed

2 files changed

+217
-6
lines changed

‎internal/pkg/otel/translate/otelconfig.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,20 +64,50 @@ func GetOtelConfig(
6464
if len(components) == 0 {
6565
return nil, nil
6666
}
67+
<<<<<<< HEAD
6768
otelConfig := confmap.New() // base config, nothing here for now
69+
=======
70+
otelConfig := confmap.New() // base config, nothing here for now
71+
extensions := map[string]bool{} // we have to manually handle extensions because otel does not merge lists, it overrides them. This is a known issue: see https://github.com/open-telemetry/opentelemetry-collector/issues/8754
72+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
6873

6974
for _, comp := range components {
7075
componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter, logger)
7176
if compErr != nil {
7277
return nil, compErr
7378
}
79+
<<<<<<< HEAD
80+
=======
81+
82+
// save the extensions, we deduplicate and add this list at the end
83+
if componentConfig.IsSet("service::extensions") {
84+
for _, extension := range componentConfig.Get("service::extensions").([]any) {
85+
extensionName := extension.(string)
86+
extensions[extensionName] = true
87+
}
88+
}
89+
90+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
7491
// the assumption here is that each component will define its own receivers, and the shared exporters
7592
// will be merged
7693
mergeErr := otelConfig.Merge(componentConfig)
7794
if mergeErr != nil {
7895
return nil, fmt.Errorf("error merging otel config for component %s: %w", comp.ID, mergeErr)
7996
}
8097
}
98+
// create a deduplicated extensions lists in a deterministic order
99+
extensionsSlice := maps.Keys(extensions)
100+
slices.Sort(extensionsSlice)
101+
// for consistency, we set this back as a slice of any
102+
untypedExtensions := make([]any, len(extensionsSlice))
103+
for i, ext := range extensionsSlice {
104+
untypedExtensions[i] = ext
105+
}
106+
extensionsConf := confmap.NewFromStringMap(map[string]any{"service::extensions": untypedExtensions})
107+
err := otelConfig.Merge(extensionsConf)
108+
if err != nil {
109+
return nil, fmt.Errorf("error merging otel extensions: %w", err)
110+
}
81111
return otelConfig, nil
82112
}
83113

‎internal/pkg/otel/translate/otelconfig_test.go‎

Lines changed: 187 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -441,16 +441,189 @@ func TestGetOtelConfig(t *testing.T) {
441441
},
442442
},
443443
"service": map[string]any{
444+
<<<<<<< HEAD
445+
=======
446+
"extensions": []any{"beatsauth/_agent-component/default"},
447+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
444448
"pipelines": map[string]any{
445449
"logs/_agent-component/filestream-default": map[string][]string{
446-
"exporters": []string{"elasticsearch/_agent-component/default"},
447-
"receivers": []string{"filebeatreceiver/_agent-component/filestream-default"},
450+
"exporters": {"elasticsearch/_agent-component/default"},
451+
"receivers": {"filebeatreceiver/_agent-component/filestream-default"},
448452
},
449453
},
450454
},
451455
}),
452456
},
453457
{
458+
<<<<<<< HEAD
459+
=======
460+
name: "multiple filestream inputs and output types",
461+
model: &component.Model{
462+
Components: []component.Component{
463+
{
464+
ID: "filestream-primaryOutput",
465+
InputType: "filestream",
466+
OutputType: "elasticsearch",
467+
InputSpec: &component.InputRuntimeSpec{
468+
BinaryName: "agentbeat",
469+
Spec: component.InputSpec{
470+
Command: &component.CommandSpec{
471+
Args: []string{"filebeat"},
472+
},
473+
},
474+
},
475+
Units: []component.Unit{
476+
{
477+
ID: "filestream-unit",
478+
Type: client.UnitTypeInput,
479+
Config: component.MustExpectedConfig(fileStreamConfig),
480+
},
481+
{
482+
ID: "filestream-primaryOutput",
483+
Type: client.UnitTypeOutput,
484+
Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.verification_mode", "certificate"})),
485+
},
486+
},
487+
},
488+
{
489+
ID: "filestream-secondaryOutput",
490+
InputType: "filestream",
491+
OutputType: "elasticsearch",
492+
InputSpec: &component.InputRuntimeSpec{
493+
BinaryName: "agentbeat",
494+
Spec: component.InputSpec{
495+
Command: &component.CommandSpec{
496+
Args: []string{"filebeat"},
497+
},
498+
},
499+
},
500+
Units: []component.Unit{
501+
{
502+
ID: "filestream-unit-2",
503+
Type: client.UnitTypeInput,
504+
Config: component.MustExpectedConfig(fileStreamConfig),
505+
},
506+
{
507+
ID: "filestream-secondaryOutput",
508+
Type: client.UnitTypeOutput,
509+
Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.ca_trusted_fingerprint", "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"})),
510+
},
511+
},
512+
},
513+
},
514+
},
515+
expectedConfig: confmap.NewFromStringMap(map[string]any{
516+
"exporters": map[string]any{
517+
"elasticsearch/_agent-component/primaryOutput": expectedESConfig("primaryOutput"),
518+
"elasticsearch/_agent-component/secondaryOutput": expectedESConfig("secondaryOutput"),
519+
},
520+
"extensions": map[string]any{
521+
"beatsauth/_agent-component/primaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"verification_mode": uint64(2)}}),
522+
"beatsauth/_agent-component/secondaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"ca_trusted_fingerprint": "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"}}),
523+
},
524+
"receivers": map[string]any{
525+
"filebeatreceiver/_agent-component/filestream-primaryOutput": expectedFilestreamConfig("filestream-primaryOutput"),
526+
"filebeatreceiver/_agent-component/filestream-secondaryOutput": expectedFilestreamConfig("filestream-secondaryOutput"),
527+
},
528+
"service": map[string]any{
529+
"extensions": []any{"beatsauth/_agent-component/primaryOutput", "beatsauth/_agent-component/secondaryOutput"},
530+
"pipelines": map[string]any{
531+
"logs/_agent-component/filestream-primaryOutput": map[string][]string{
532+
"exporters": {"elasticsearch/_agent-component/primaryOutput"},
533+
"receivers": {"filebeatreceiver/_agent-component/filestream-primaryOutput"},
534+
},
535+
"logs/_agent-component/filestream-secondaryOutput": map[string][]string{
536+
"exporters": {"elasticsearch/_agent-component/secondaryOutput"},
537+
"receivers": {"filebeatreceiver/_agent-component/filestream-secondaryOutput"},
538+
},
539+
},
540+
},
541+
}),
542+
},
543+
{
544+
name: "multiple filestream inputs and one output",
545+
model: &component.Model{
546+
Components: []component.Component{
547+
{
548+
ID: "filestream1-default",
549+
InputType: "filestream",
550+
OutputType: "elasticsearch",
551+
InputSpec: &component.InputRuntimeSpec{
552+
BinaryName: "agentbeat",
553+
Spec: component.InputSpec{
554+
Command: &component.CommandSpec{
555+
Args: []string{"filebeat"},
556+
},
557+
},
558+
},
559+
Units: []component.Unit{
560+
{
561+
ID: "filestream-unit",
562+
Type: client.UnitTypeInput,
563+
Config: component.MustExpectedConfig(fileStreamConfig),
564+
},
565+
{
566+
ID: "filestream-default",
567+
Type: client.UnitTypeOutput,
568+
Config: component.MustExpectedConfig(esOutputConfig()),
569+
},
570+
},
571+
},
572+
{
573+
ID: "filestream2-default",
574+
InputType: "filestream",
575+
OutputType: "elasticsearch",
576+
InputSpec: &component.InputRuntimeSpec{
577+
BinaryName: "agentbeat",
578+
Spec: component.InputSpec{
579+
Command: &component.CommandSpec{
580+
Args: []string{"filebeat"},
581+
},
582+
},
583+
},
584+
Units: []component.Unit{
585+
{
586+
ID: "filestream-unit",
587+
Type: client.UnitTypeInput,
588+
Config: component.MustExpectedConfig(fileStreamConfig),
589+
},
590+
{
591+
ID: "filestream-default",
592+
Type: client.UnitTypeOutput,
593+
Config: component.MustExpectedConfig(esOutputConfig()),
594+
},
595+
},
596+
},
597+
},
598+
},
599+
expectedConfig: confmap.NewFromStringMap(map[string]any{
600+
"exporters": map[string]any{
601+
"elasticsearch/_agent-component/default": expectedESConfig("default"),
602+
},
603+
"extensions": map[string]any{
604+
"beatsauth/_agent-component/default": expectedExtensionConfig(),
605+
},
606+
"receivers": map[string]any{
607+
"filebeatreceiver/_agent-component/filestream1-default": expectedFilestreamConfig("filestream1-default"),
608+
"filebeatreceiver/_agent-component/filestream2-default": expectedFilestreamConfig("filestream2-default"),
609+
},
610+
"service": map[string]any{
611+
"extensions": []any{"beatsauth/_agent-component/default"},
612+
"pipelines": map[string]any{
613+
"logs/_agent-component/filestream1-default": map[string][]string{
614+
"exporters": {"elasticsearch/_agent-component/default"},
615+
"receivers": {"filebeatreceiver/_agent-component/filestream1-default"},
616+
},
617+
"logs/_agent-component/filestream2-default": map[string][]string{
618+
"exporters": {"elasticsearch/_agent-component/default"},
619+
"receivers": {"filebeatreceiver/_agent-component/filestream2-default"},
620+
},
621+
},
622+
},
623+
}),
624+
},
625+
{
626+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
454627
name: "beat/metrics",
455628
model: &component.Model{
456629
Components: []component.Component{
@@ -534,10 +707,14 @@ func TestGetOtelConfig(t *testing.T) {
534707
},
535708
},
536709
"service": map[string]any{
710+
<<<<<<< HEAD
711+
=======
712+
"extensions": []any{"beatsauth/_agent-component/default"},
713+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
537714
"pipelines": map[string]any{
538715
"logs/_agent-component/beat-metrics-monitoring": map[string][]string{
539-
"exporters": []string{"elasticsearch/_agent-component/default"},
540-
"receivers": []string{"metricbeatreceiver/_agent-component/beat-metrics-monitoring"},
716+
"exporters": {"elasticsearch/_agent-component/default"},
717+
"receivers": {"metricbeatreceiver/_agent-component/beat-metrics-monitoring"},
541718
},
542719
},
543720
},
@@ -638,10 +815,14 @@ func TestGetOtelConfig(t *testing.T) {
638815
},
639816
},
640817
"service": map[string]any{
818+
<<<<<<< HEAD
819+
=======
820+
"extensions": []any{"beatsauth/_agent-component/default"},
821+
>>>>>>> 5aacbcbaa (Deduplicate extensions when generating otel config (#10172))
641822
"pipelines": map[string]any{
642823
"logs/_agent-component/system-metrics": map[string][]string{
643-
"exporters": []string{"elasticsearch/_agent-component/default"},
644-
"receivers": []string{"metricbeatreceiver/_agent-component/system-metrics"},
824+
"exporters": {"elasticsearch/_agent-component/default"},
825+
"receivers": {"metricbeatreceiver/_agent-component/system-metrics"},
645826
},
646827
},
647828
},

0 commit comments

Comments
 (0)