diff --git a/README.md b/README.md index eef3657..cb2e4cf 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Application Options: --azure.servicediscovery.cache= Duration for caching Azure ServiceDiscovery of workspaces to reduce API calls (time.Duration) (default: 30m) [$AZURE_SERVICEDISCOVERY_CACHE] --metrics.resourceid.lowercase Publish lowercase Azure Resoruce ID in metrics [$METRIC_RESOURCEID_LOWERCASE] + --metrics.set-timestamp Set timestamp on scraped metrics [$METRIC_SET_TIMESTAMP] --metrics.template= Template for metric name (default: {name}) [$METRIC_TEMPLATE] --metrics.help= Metric help (with template support) (default: Azure monitor insight metric) [$METRIC_HELP] diff --git a/config/opts.go b/config/opts.go index e224d40..f3f0453 100644 --- a/config/opts.go +++ b/config/opts.go @@ -26,6 +26,7 @@ type ( Metrics struct { ResourceIdLowercase bool `long:"metrics.resourceid.lowercase" env:"METRIC_RESOURCEID_LOWERCASE" description:"Publish lowercase Azure Resoruce ID in metrics"` + SetTimestamp bool `long:"metrics.set-timestamp" env:"METRIC_SET_TIMESTAMP" description:"Set timestamp on scraped metrics"` Template string `long:"metrics.template" env:"METRIC_TEMPLATE" description:"Template for metric name" default:"{name}"` Help string `long:"metrics.help" env:"METRIC_HELP" description:"Metric help (with template support)" default:"Azure monitor insight metric"` } diff --git a/go.mod b/go.mod index 01762d2..959a3c5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,10 @@ require ( google.golang.org/protobuf v1.27.1 // indirect ) -require github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7 +require ( + github.com/google/go-cmp v0.5.5 + github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7 +) require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/metrics/collector.go b/metrics/collector.go new file mode 100644 index 0000000..236674b --- /dev/null +++ b/metrics/collector.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +type metricListCollectorDetails struct { + gauge prometheus.Gauge + desc *prometheus.Desc + ts time.Time +} + +type metricListCollector struct { + details []*metricListCollectorDetails +} + +func NewMetricListCollector(list *MetricList) *metricListCollector { + collector := &metricListCollector{ + details: []*metricListCollectorDetails{}, + } + + if list == nil { + return collector + } + + // create prometheus metrics and set rows + for _, metricName := range list.GetMetricNames() { + gaugeVec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: metricName, + Help: list.GetMetricHelp(metricName), + }, + list.GetMetricLabelNames(metricName), + ) + + for _, metric := range list.GetUniqueMetricList(metricName) { + gauge := gaugeVec.With(metric.Labels) + gauge.Set(metric.Value) + + desc := prometheus.NewDesc(metricName, list.GetMetricHelp(metricName), []string{}, metric.Labels) + + details := &metricListCollectorDetails{ + gauge, + desc, + metric.Timestamp, + } + + collector.details = append(collector.details, details) + } + } + + return collector +} + +func (c *metricListCollector) Describe(ch chan<- *prometheus.Desc) { + for _, detail := range c.details { + ch <- detail.desc + } +} + +func (c *metricListCollector) Collect(ch chan<- prometheus.Metric) { + for _, detail := range c.details { + s := prometheus.NewMetricWithTimestamp(detail.ts, detail.gauge) + ch <- s + } +} diff --git a/metrics/insights.go b/metrics/insights.go index 782a6b0..9c3241a 100644 --- a/metrics/insights.go +++ b/metrics/insights.go @@ -1,12 +1,14 @@ package metrics import ( + "regexp" + "strings" + "time" + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" "github.com/prometheus/client_golang/prometheus" - "regexp" - "strings" ) var ( @@ -26,10 +28,11 @@ type ( } PrometheusMetricResult struct { - Name string - Labels prometheus.Labels - Value float64 - Help string + Name string + Labels prometheus.Labels + Value float64 + Timestamp time.Time + Help string } ) @@ -70,7 +73,7 @@ func (p *MetricProber) FetchMetricsFromTarget(client *insights.MetricsClient, ta return ret, err } -func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64) (metric PrometheusMetricResult) { +func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64, timestamp time.Time) (metric PrometheusMetricResult) { // copy map to ensure we don't keep references metricLabels := prometheus.Labels{} for labelName, labelValue := range labels { @@ -78,9 +81,10 @@ func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value } metric = PrometheusMetricResult{ - Name: r.settings.MetricTemplate, - Labels: metricLabels, - Value: value, + Name: r.settings.MetricTemplate, + Labels: metricLabels, + Value: value, + Timestamp: timestamp, } // fallback if template is empty (should not be) @@ -140,8 +144,8 @@ func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- PrometheusMetricResult) { if r.Result.Value != nil { // DEBUGGING - //data, _ := json.Marshal(r.Result) - //fmt.Println(string(data)) + // data, _ := json.Marshal(r.Result) + // fmt.Println(string(data)) for _, metric := range *r.Result.Value { if metric.Timeseries != nil { @@ -203,6 +207,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Total, + timeseriesData.TimeStamp.Time, ) } @@ -211,6 +216,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Minimum, + timeseriesData.TimeStamp.Time, ) } @@ -219,6 +225,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Maximum, + timeseriesData.TimeStamp.Time, ) } @@ -227,6 +234,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Average, + timeseriesData.TimeStamp.Time, ) } @@ -235,6 +243,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Count, + timeseriesData.TimeStamp.Time, ) } } diff --git a/metrics/metrics.go b/metrics/metrics.go index 1a32359..c5b4c7d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,6 +1,11 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "time" + + "github.com/google/go-cmp/cmp" + "github.com/prometheus/client_golang/prometheus" +) const ( MetricHelpDefault = "Azure monitor insight metric" @@ -13,8 +18,9 @@ type ( } MetricRow struct { - Labels prometheus.Labels - Value float64 + Labels prometheus.Labels + Value float64 + Timestamp time.Time } ) @@ -55,6 +61,30 @@ func (l *MetricList) GetMetricList(name string) []MetricRow { return l.List[name] } +func (l *MetricList) GetUniqueMetricList(name string) []MetricRow { + rows := []MetricRow{} + + for _, row := range l.List[name] { + existed := false + + for idx, existedRow := range rows { + if cmp.Equal(row.Labels, existedRow.Labels) { + existed = true + + if row.Timestamp.After(existedRow.Timestamp) { + rows[idx] = row + } + } + } + + if !existed { + rows = append(rows, row) + } + } + + return rows +} + func (l *MetricList) GetMetricLabelNames(name string) []string { var list []string uniqueLabelMap := map[string]string{} diff --git a/metrics/prober.go b/metrics/prober.go index 8610f7c..426d067 100644 --- a/metrics/prober.go +++ b/metrics/prober.go @@ -211,8 +211,9 @@ func (p *MetricProber) collectMetricsFromTargets() { for result := range metricsChannel { metric := MetricRow{ - Labels: result.Labels, - Value: result.Value, + Labels: result.Labels, + Value: result.Value, + Timestamp: result.Timestamp, } p.metricList.Add(result.Name, metric) p.metricList.SetMetricHelp(result.Name, result.Help) @@ -220,6 +221,11 @@ func (p *MetricProber) collectMetricsFromTargets() { } func (p *MetricProber) publishMetricList() { + if p.Conf.Metrics.SetTimestamp { + p.prometheus.registry.MustRegister(NewMetricListCollector(p.metricList)) + return + } + if p.metricList == nil { return }