Adding go memstat metrics

.devcontainer/Dockerfile

@@ -96,8 +96,8 @@ RUN apt-get update -y \
     && apt-get -y install --no-install-recommends apt-utils 2>&1 \
     # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed.
     && apt-get -y install git iproute2 procps lsb-release \
-    # Install Python2.7
-    && apt-get install -y python2.7 python-pip unzip \
+    # Install Python
+    && apt-get install -y python3 python3-pip unzip \
     && apt-get install -y protobuf-compiler \
     && apt-get autoremove -y \
     && apt-get clean -y \

cmd/agent/main_common.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"time"
 
+	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
+	"github.com/StackVista/stackstate-process-agent/telemetry"
+
 	"github.com/StackVista/stackstate-agent/pkg/pidfile"
 	log "github.com/cihub/seelog"
 
@@ -172,6 +174,12 @@ func runAgent(exit chan bool) {
 		return
 	}
 
+	if cfg.OpenMetricsEnabled {
+		// Expose the registered metrics via HTTP.
+		http.Handle("/metrics", telemetry.Handler())
+		go http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", cfg.OpenMetricsPort), nil) //nolint:errcheck
+	}
+
 	if opts.info {
 		// using the debug port to get info to work
 		url := "http://localhost:6062/debug/vars"

config/config.go

@@ -103,6 +103,10 @@ type AgentConfig struct {
 	EnableShortLivedProcessFilter  bool
 	ShortLivedProcessQualifierSecs time.Duration
 
+	// Expose agent metrics via openmetrics endpoint
+	OpenMetricsEnabled bool
+	OpenMetricsPort    int
+
 	// Relation Cache Expiration, In Minutes
 	NetworkRelationCacheDurationMin time.Duration
 
@@ -232,6 +236,9 @@ func NewDefaultAgentConfig() *AgentConfig {
 		DDAgentPy:    defaultDDAgentPy,
 		DDAgentPyEnv: []string{defaultDDAgentPyEnv},
 
+		OpenMetricsEnabled: false,
+		OpenMetricsPort:    5123,
+
 		EnableIncrementalPublishing:          true,
 		IncrementalPublishingRefreshInterval: 1 * time.Minute,
 
@@ -764,6 +771,13 @@ func mergeEnvironmentVariables(c *AgentConfig) *AgentConfig {
 		setNetworkRelationFilters(c, true, v)
 	}
 
+	if ok, _ := isAffirmative(os.Getenv("STS_PROCESS_AGENT_OPEN_METRICS_ENABLED")); ok {
+		c.OpenMetricsEnabled = true
+	}
+
+	if v, err := strconv.Atoi(os.Getenv("STS_PROCESS_AGENT_OPEN_METRICS_PORT")); err == nil {
+		c.OpenMetricsPort = v
+	}
 	return c
 }
 

config/yaml_config.go

@@ -23,6 +23,10 @@ type YamlAgentConfig struct {
 	StsURL string `yaml:"sts_url"`
 	// Whether or not the process-agent should output logs to console
 	LogToConsole bool `yaml:"log_to_console"`
+
+	OpenMetricsEnabled bool `yaml:"open_metrics_enabled"`
+	OpenMetricsPort    int  `yaml:"open_metrics_port"`
+
 	// Incremental publishing: send only changes to server, instead of snapshots
 	IncrementalPublishingEnabled string `yaml:"incremental_publishing_enabled"`
 	// Periodically resend all data to allow downstream to recover from any lost data
@@ -188,6 +192,13 @@ func mergeYamlConfig(agentConf *AgentConfig, yc *YamlAgentConfig) (*AgentConfig,
 		agentConf.LogFile = yc.Process.LogFile
 	}
 
+	if yc.OpenMetricsEnabled {
+		agentConf.OpenMetricsEnabled = true
+	}
+	if yc.OpenMetricsPort != 0 {
+		agentConf.OpenMetricsPort = yc.OpenMetricsPort
+	}
+
 	// (Re)configure the logging from our configuration
 	if err := NewLoggerLevel(agentConf.LogLevel, agentConf.LogFile, agentConf.LogToConsole); err != nil {
 		return nil, err

packaging/publish_image.sh

@@ -17,7 +17,7 @@ cp $ARTIFACT_PATH DockerFiles/agent
 docker build -t stackstate/${IMAGE_REPO}:${IMAGE_TAG} DockerFiles/agent
 
 
-docker login -u $DOCKER_USER -p $DOCKER_PASS
+docker login -u $docker_user -p $docker_password
 docker push stackstate/${IMAGE_REPO}:${IMAGE_TAG}
 
 if [ "$PUSH_LATEST" = "true" ]; then

telemetry/counter.go

@@ -0,0 +1,65 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-2020 Datadog, Inc.
+
+package telemetry
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Counter tracks how many times something is happening.
+type Counter interface {
+	// Inc increments the counter with the given tags value.
+	Inc(tagsValue ...string)
+	// Add adds the given value to the counter with the given tags value.
+	Add(value float64, tagsValue ...string)
+	// Delete deletes the value for the counter with the given tags value.
+	Delete(tagsValue ...string)
+	// IncWithTags increments the counter with the given tags.
+	// Even if less convenient, this signature could be used in hot path
+	// instead of Inc(...string) to avoid escaping the parameters on the heap.
+	IncWithTags(tags map[string]string)
+	// AddWithTags adds the given value to the counter with the given tags.
+	// Even if less convenient, this signature could be used in hot path
+	// instead of Add(float64, ...string) to avoid escaping the parameters on the heap.
+	AddWithTags(value float64, tags map[string]string)
+	// DeleteWithTags deletes the value for the counter with the given tags.
+	// Even if less convenient, this signature could be used in hot path
+	// instead of Delete(...string) to avoid escaping the parameters on the heap.
+	DeleteWithTags(tags map[string]string)
+}
+
+// NewCounter creates a Counter with default options for telemetry purpose.
+// Current implementation used: Prometheus Counter
+func NewCounter(subsystem, name string, tags []string, help string) Counter {
+	return NewCounterWithOpts(subsystem, name, tags, help, DefaultOptions)
+}
+
+// NewCounterWithOpts creates a Counter with the given options for telemetry purpose.
+// See NewCounter()
+func NewCounterWithOpts(subsystem, name string, tags []string, help string, opts Options) Counter {
+	// subsystem is optional
+	if subsystem != "" && !opts.NoDoubleUnderscoreSep {
+		// Prefix metrics with a _, prometheus will add a second _
+		// It will create metrics with a custom separator and
+		// will let us replace it to a dot later in the process.
+		name = fmt.Sprintf("_%s", name)
+	}
+
+	c := &promCounter{
+		pc: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Subsystem: subsystem,
+				Name:      name,
+				Help:      help,
+			},
+			tags,
+		),
+	}
+	telemetryRegistry.MustRegister(c.pc)
+	return c
+}

telemetry/gauge.go

@@ -0,0 +1,59 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-2020 Datadog, Inc.
+
+package telemetry
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Gauge tracks the value of one health metric of the Agent.
+type Gauge interface {
+	// Set stores the value for the given tags.
+	Set(value float64, tagsValue ...string)
+	// Inc increments the Gauge value.
+	Inc(tagsValue ...string)
+	// Dec decrements the Gauge value.
+	Dec(tagsValue ...string)
+	// Add adds the value to the Gauge value.
+	Add(value float64, tagsValue ...string)
+	// Sub subtracts the value to the Gauge value.
+	Sub(value float64, tagsValue ...string)
+	// Delete deletes the value for the Gauge with the given tags.
+	Delete(tagsValue ...string)
+}
+
+// NewGauge creates a Gauge with default options for telemetry purpose.
+// Current implementation used: Prometheus Gauge
+func NewGauge(subsystem, name string, tags []string, help string) Gauge {
+	return NewGaugeWithOpts(subsystem, name, tags, help, DefaultOptions)
+}
+
+// NewGaugeWithOpts creates a Gauge with the given options for telemetry purpose.
+// See NewGauge()
+func NewGaugeWithOpts(subsystem, name string, tags []string, help string, opts Options) Gauge {
+	// subsystem is optional
+	if subsystem != "" && !opts.NoDoubleUnderscoreSep {
+		// Prefix metrics with a _, prometheus will add a second _
+		// It will create metrics with a custom separator and
+		// will let us replace it to a dot later in the process.
+		name = fmt.Sprintf("_%s", name)
+	}
+
+	g := &promGauge{
+		pg: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Subsystem: subsystem,
+				Name:      name,
+				Help:      help,
+			},
+			tags,
+		),
+	}
+	telemetryRegistry.MustRegister(g.pg)
+	return g
+}

telemetry/options.go

@@ -0,0 +1,17 @@
+package telemetry
+
+// Options for telemetry metrics.
+// Creating an Options struct without specifying any of its fields should be the
+// equivalent of using the DefaultOptions var.
+type Options struct {
+	// NoDoubleUnderscoreSep is set to true when you don't want to
+	// separate the subsystem and the name with a double underscore separator.
+	NoDoubleUnderscoreSep bool
+}
+
+// DefaultOptions for telemetry metrics which don't need to specify any option.
+var DefaultOptions = Options{
+	// By default, we want to separate the subsystem and the metric name with a
+	// double underscore to be able to replace it later in the process.
+	NoDoubleUnderscoreSep: false,
+}

telemetry/prom_counter.go

@@ -0,0 +1,51 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-2020 Datadog, Inc.
+
+package telemetry
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Counter implementation using Prometheus.
+type promCounter struct {
+	pc *prometheus.CounterVec
+}
+
+// Add adds the given value to the counter with the given tags value.
+func (c *promCounter) Add(value float64, tagsValue ...string) {
+	c.pc.WithLabelValues(tagsValue...).Add(value)
+}
+
+// AddWithTags adds the given value to the counter with the given tags.
+// Even if less convenient, this signature could be used in hot path
+// instead of Add(float64, ...string) to avoid escaping the parameters on the heap.
+func (c *promCounter) AddWithTags(value float64, tags map[string]string) {
+	c.pc.With(tags).Add(value)
+}
+
+// Inc increments the counter with the given tags value.
+func (c *promCounter) Inc(tagsValue ...string) {
+	c.pc.WithLabelValues(tagsValue...).Inc()
+}
+
+// IncWithTags increments the counter with the given tags.
+// Even if less convenient, this signature could be used in hot path
+// instead of Inc(...string) to avoid escaping the parameters on the heap.
+func (c *promCounter) IncWithTags(tags map[string]string) {
+	c.pc.With(tags).Inc()
+}
+
+// Delete deletes the value for the counter with the given tags value.
+func (c *promCounter) Delete(tagsValue ...string) {
+	c.pc.DeleteLabelValues(tagsValue...)
+}
+
+// DeleteWithTags deletes the value for the counter with the given tags.
+// Even if less convenient, this signature could be used in hot path
+// instead of Delete(...string) to avoid escaping the parameters on the heap.
+func (c *promCounter) DeleteWithTags(tags map[string]string) {
+	c.pc.Delete(tags)
+}

telemetry/prom_gauge.go

@@ -0,0 +1,45 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-2020 Datadog, Inc.
+
+package telemetry
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Gauge implementation using Prometheus.
+type promGauge struct {
+	pg *prometheus.GaugeVec
+}
+
+// Set stores the value for the given tags.
+func (g *promGauge) Set(value float64, tagsValue ...string) {
+	g.pg.WithLabelValues(tagsValue...).Set(value)
+}
+
+// Inc increments the Gauge value.
+func (g *promGauge) Inc(tagsValue ...string) {
+	g.pg.WithLabelValues(tagsValue...).Inc()
+}
+
+// Dec decrements the Gauge value.
+func (g *promGauge) Dec(tagsValue ...string) {
+	g.pg.WithLabelValues(tagsValue...).Dec()
+}
+
+// Delete deletes the value for the Gauge with the given tags.
+func (g *promGauge) Delete(tagsValue ...string) {
+	g.pg.DeleteLabelValues(tagsValue...)
+}
+
+// Add adds the value to the Gauge value.
+func (g *promGauge) Add(value float64, tagsValue ...string) {
+	g.pg.WithLabelValues(tagsValue...).Add(value)
+}
+
+// Sub subtracts the value to the Gauge value.
+func (g *promGauge) Sub(value float64, tagsValue ...string) {
+	g.pg.WithLabelValues(tagsValue...).Sub(value)
+}

telemetry/prom_telemetry.go

@@ -0,0 +1,28 @@
+package telemetry
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+	telemetryRegistry = prometheus.NewRegistry()
+)
+
+func init() {
+	telemetryRegistry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	telemetryRegistry.MustRegister(prometheus.NewGoCollector())
+}
+
+// Handler serves the HTTP route containing the prometheus metrics.
+func Handler() http.Handler {
+	return promhttp.HandlerFor(telemetryRegistry, promhttp.HandlerOpts{})
+}
+
+// Reset resets the global telemetry registry, stopping the collection of every previously registered metrics.
+// Mainly used for unit tests and integration tests.
+func Reset() {
+	telemetryRegistry = prometheus.NewRegistry()
+}