feat: tool to build custom Telegraf builds (influxdata#11524)

.gitignore

@@ -6,6 +6,7 @@
 /tools/package_lxd_test/package_lxd_test
 /tools/license_checker/license_checker*
 /tools/readme_config_includer/generator*
+/tools/custom_builder/custom_builder*
 /vendor
 .DS_Store
 process.yml

.golangci.yml

@@ -126,7 +126,7 @@ issues:
     - path: _test\.go
       text: "parameter.*seems to be a control flag, avoid control coupling"
 
-    - path: (^agent/|^cmd/|^config/|^filter/|^internal/|^logger/|^metric/|^models/|^selfstat/|^testutil/|^plugins/serializers/|^plugins/inputs/zipkin/cmd)
+    - path: (^agent/|^cmd/|^config/|^filter/|^internal/|^logger/|^metric/|^models/|^selfstat/|^testutil/|^tools|^plugins/serializers/|^plugins/inputs/zipkin/cmd)
       text: "imports-blacklist: should not use the following blacklisted import: \"log\""
       linters:
         - revive

Makefile

@@ -114,6 +114,7 @@ versioninfo:
 	go generate cmd/telegraf/telegraf_windows.go; \
 
 build_tools:
+	$(HOSTGO) build -o ./tools/custom_builder/custom_builder$(EXEEXT) ./tools/custom_builder
 	$(HOSTGO) build -o ./tools/license_checker/license_checker$(EXEEXT) ./tools/license_checker
 	$(HOSTGO) build -o ./tools/readme_config_includer/generator$(EXEEXT) ./tools/readme_config_includer/generator.go
 
@@ -223,6 +224,8 @@ clean:
 	rm -f telegraf
 	rm -f telegraf.exe
 	rm -rf build
+	rm -rf tools/custom_builder/custom_builder
+	rm -rf tools/custom_builder/custom_builder.exe
 	rm -rf tools/readme_config_includer/generator
 	rm -rf tools/readme_config_includer/generator.exe
 	rm -rf tools/package_lxd_test/package_lxd_test

cmd/telegraf/telegraf.go

@@ -27,12 +27,15 @@ import (
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/goplugin"
 	"github.com/influxdata/telegraf/logger"
+	"github.com/influxdata/telegraf/plugins/aggregators"
 	_ "github.com/influxdata/telegraf/plugins/aggregators/all"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	"github.com/influxdata/telegraf/plugins/outputs"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
+	"github.com/influxdata/telegraf/plugins/parsers"
 	_ "github.com/influxdata/telegraf/plugins/parsers/all"
+	"github.com/influxdata/telegraf/plugins/processors"
 	_ "github.com/influxdata/telegraf/plugins/processors/all"
 	"gopkg.in/tomb.v1"
 )
@@ -271,7 +274,14 @@ func runAgent(ctx context.Context,
 
 	logger.SetupLogging(logConfig)
 
-	log.Printf("I! Starting Telegraf %s", internal.Version())
+	log.Printf("I! Starting Telegraf %s%s", internal.Version(), internal.Customized)
+	log.Printf("I! Available plugins: %d inputs, %d aggregators, %d processors, %d parsers, %d outputs",
+		len(inputs.Inputs),
+		len(aggregators.Aggregators),
+		len(processors.Processors),
+		len(parsers.Parsers),
+		len(outputs.Outputs),
+	)
 	log.Printf("I! Loaded inputs: %s", strings.Join(c.InputNames(), " "))
 	log.Printf("I! Loaded aggregators: %s", strings.Join(c.AggregatorNames(), " "))
 	log.Printf("I! Loaded processors: %s", strings.Join(c.ProcessorNames(), " "))

config/config.go

@@ -386,7 +386,7 @@ func (c *Config) LoadConfig(path string) error {
 			return err
 		}
 	}
-	data, err := loadConfig(path)
+	data, err := LoadConfigFile(path)
 	if err != nil {
 		return fmt.Errorf("Error loading config file %s: %w", path, err)
 	}
@@ -565,7 +565,7 @@ func escapeEnv(value string) string {
 	return envVarEscaper.Replace(value)
 }
 
-func loadConfig(config string) ([]byte, error) {
+func LoadConfigFile(config string) ([]byte, error) {
 	if fetchURLRe.MatchString(config) {
 		u, err := url.Parse(config)
 		if err != nil {

docs/CUSTOMIZATION.md

@@ -17,13 +17,13 @@ build as otherwise _all_ plugins will be selected regardless of other tags.
 ## Via make
 
 When using the project's makefile, the build can be customized via the
-`BUILDTAGS` environment variable containing a __space-separated__ list of the
+`BUILDTAGS` environment variable containing a __comma-separated__ list of the
 selected plugins (or categories) __and__ the `custom` tag.
 
 For example
 
 ```shell
-BUILDTAGS="custom inputs outputs.influxdb_v2 parsers.json" make
+BUILDTAGS="custom,inputs,outputs.influxdb_v2,parsers.json" make
 ```
 
 will build a customized Telegraf including _all_ `inputs`, the InfluxDB v2
@@ -32,13 +32,13 @@ will build a customized Telegraf including _all_ `inputs`, the InfluxDB v2
 ## Via `go build`
 
 If you wish to build Telegraf using native go tools, you can use the `go build`
-command with the `-tags` option. Specify  a __space-separated__ list of the
+command with the `-tags` option. Specify  a __comma-separated__ list of the
 selected plugins (or categories) __and__ the `custom` tag as argument.
 
 For example
 
 ```shell
-go build -tags "custom inputs outputs.influxdb_v2 parsers.json" ./cmd/telegraf
+go build -tags "custom,inputs,outputs.influxdb_v2,parsers.json" ./cmd/telegraf
 ```
 
 will build a customized Telegraf including _all_ `inputs`, the InfluxDB v2

filter/filter.go

@@ -45,6 +45,14 @@ func Compile(filters []string) (Filter, error) {
 	}
 }
 
+func MustCompile(filters []string) Filter {
+	f, err := Compile(filters)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
 // hasMeta reports whether path contains any magic glob characters.
 func hasMeta(s string) bool {
 	return strings.ContainsAny(s, "*?[")

internal/customized_no.go

@@ -0,0 +1,5 @@
+//go:build !custom
+
+package internal
+
+const Customized = ""

internal/customized_yes.go

@@ -0,0 +1,5 @@
+//go:build custom
+
+package internal
+
+const Customized = " (customized)"

tools/custom_builder/README.md

@@ -0,0 +1,81 @@
+# Telegraf customization tool
+
+Telegraf's `custom_builder` is a tool to select the plugins compiled into the
+Telegraf binary. By doing so, Telegraf can become smaller, saving both disk
+space and memory if only a sub-set of plugins is selected.
+
+## Building
+
+To build `custom_builder` run the following command:
+
+```shell
+# make build_tools
+```
+
+The resulting binary is located in the `tools/custom_builder` folder.
+
+## Running
+
+The easiest way of building a customized Telegraf is to use your
+Telegraf configuration file(s). Assuming your configuration is
+in `/etc/telegraf/telegraf.conf` you can run
+
+```shell
+# ./tools/custom_builder/custom_builder --config /etc/telegraf/telegraf.conf
+```
+
+to build a Telegraf binary tailored to your configuration.
+You can also specify a configuration directory similar to
+Telegraf itself. To additionally use the configurations in
+`/etc/telegraf/telegraf.d` run
+
+```shell
+# ./tools/custom_builder/custom_builder                      \
+    --config     /etc/telegraf/telegraf.conf \
+    --config-dir /etc/telegraf/telegraf.d
+```
+
+Configurations can also be retrieved from remote locations just
+like for Telegraf.
+
+```shell
+# ./tools/custom_builder/custom_builder --config http://myserver/telegraf.conf
+```
+
+will download the configuration from `myserver`.
+
+The `--config` and `--config-dir` option can be used multiple times.
+In case you want to deploy Telegraf to multiple systems with
+different configurations, simply specify the super-set of all
+configurations you have. `custom_builder` will figure out the list
+for you
+
+```shell
+# ./tools/custom_builder/custom_builder             \
+    --config system1/telegraf.conf  \
+    --config system2/telegraf.conf  \
+    --config ...                    \
+    --config systemN/telegraf.conf  \
+    --config-dir system1/telegraf.d \
+    --config-dir system2/telegraf.d \
+    --config-dir ...                \
+    --config-dir systemN/telegraf.d
+```
+
+The Telegraf customization uses
+[Golang's build-tags](https://pkg.go.dev/go/build#hdr-Build_Constraints) to
+select the set of plugins. To see which tags are set use the `--tags` flag.
+
+To get more help run
+
+```shell
+# ./tools/custom_builder/custom_builder --help
+```
+
+## Notes
+
+Please make sure to include all `parsers` you intend to use and check the
+enabled-plugins list.
+
+Additional plugins can potentially be enabled automatically due to
+dependencies without being shown in the enabled-plugins list.

tools/custom_builder/config.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/toml"
+	"github.com/influxdata/toml/ast"
+)
+
+type pluginState map[string]bool
+type selection map[string]pluginState
+
+func ImportConfigurations(files, dirs []string) (*selection, int, error) {
+	sel := selection(make(map[string]pluginState))
+
+	// Initialize the categories
+	for _, category := range categories {
+		sel[category] = make(map[string]bool)
+	}
+
+	// Gather all configuration files
+	var filenames []string
+	filenames = append(filenames, files...)
+
+	for _, dir := range dirs {
+		// Walk the directory and get the packages
+		elements, err := os.ReadDir(dir)
+		if err != nil {
+			return nil, 0, fmt.Errorf("reading directory %q failed: %w", dir, err)
+		}
+
+		for _, element := range elements {
+			if element.IsDir() || filepath.Ext(element.Name()) != ".conf" {
+				continue
+			}
+
+			filenames = append(filenames, filepath.Join(dir, element.Name()))
+		}
+	}
+	if len(filenames) == 0 {
+		return &sel, 0, errors.New("no configuration files given or found")
+	}
+
+	// Do the actual import
+	err := sel.importFiles(filenames)
+	return &sel, len(filenames), err
+}
+
+func (s *selection) Filter(p packageCollection) (*packageCollection, error) {
+	enabled := packageCollection{
+		packages: map[string][]packageInfo{},
+	}
+
+	for category, pkgs := range p.packages {
+		var categoryEnabledPackages []packageInfo
+		settings := (*s)[category]
+		for _, pkg := range pkgs {
+			if _, found := settings[pkg.Plugin]; found {
+				categoryEnabledPackages = append(categoryEnabledPackages, pkg)
+			}
+		}
+		enabled.packages[category] = categoryEnabledPackages
+	}
+
+	// Make sure we update the list of default parsers used by
+	// the remaining packages
+	enabled.FillDefaultParsers()
+
+	// If the user did not configure any parser, we want to include
+	// the default parsers if any to preserve a functional set of
+	// plugins.
+	if len(enabled.packages["parsers"]) == 0 && len(enabled.defaultParsers) > 0 {
+		var parsers []packageInfo
+		for _, pkg := range p.packages["parsers"] {
+			for _, name := range enabled.defaultParsers {
+				if pkg.Plugin == name {
+					parsers = append(parsers, pkg)
+					break
+				}
+			}
+		}
+		enabled.packages["parsers"] = parsers
+	}
+
+	return &enabled, nil
+}
+
+func (s *selection) importFiles(configurations []string) error {
+	for _, cfg := range configurations {
+		buf, err := config.LoadConfigFile(cfg)
+		if err != nil {
+			return fmt.Errorf("reading %q failed: %v", cfg, err)
+		}
+
+		if err := s.extractPluginsFromConfig(buf); err != nil {
+			return fmt.Errorf("extracting plugins from %q failed: %v", cfg, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *selection) extractPluginsFromConfig(buf []byte) error {
+	table, err := toml.Parse(trimBOM(buf))
+	if err != nil {
+		return fmt.Errorf("parsing TOML failed: %w", err)
+	}
+
+	for category, subtbl := range table.Fields {
+		categoryTbl, ok := subtbl.(*ast.Table)
+		if !ok {
+			continue
+		}
+
+		if _, found := (*s)[category]; !found {
+			continue
+		}
+
+		for name, data := range categoryTbl.Fields {
+			(*s)[category][name] = true
+
+			// We need to check the data_format field to get all required parsers
+			switch category {
+			case "inputs", "processors":
+				pluginTables, ok := data.([]*ast.Table)
+				if !ok {
+					continue
+				}
+				for _, subsubtbl := range pluginTables {
+					for field, fieldData := range subsubtbl.Fields {
+						if field != "data_format" {
+							continue
+						}
+						kv := fieldData.(*ast.KeyValue)
+						name := kv.Value.(*ast.String)
+						(*s)["parsers"][name.Value] = true
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func trimBOM(f []byte) []byte {
+	return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
+}