Skip to content

Commit d8edc95

Browse files
pracuccipracucci
authored
Basic functional options support to metrics assertion in integration tests (#2522)
* Basic functional options support to metrics assertion in integration tests Signed-off-by: Marco Pracucci <[email protected]> * Added WithLabelMatchers() option to integration tests Signed-off-by: Marco Pracucci <[email protected]> * Removed time.Sleep() from TestAlertmanagerStoreAPI Signed-off-by: Marco Pracucci <[email protected]> * Removed last time.Sleep() from integration tests Signed-off-by: Marco Pracucci <[email protected]> * Renamed WaitMissingMetric() into WaitRemovedMetric() Signed-off-by: Marco Pracucci <[email protected]>
1 parent b404a95 commit d8edc95

10 files changed

+222
-132
lines changed

‎integration/alertmanager_test.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ package main
55
import (
66
"context"
77
"testing"
8-
"time"
98

9+
"github.com/prometheus/prometheus/pkg/labels"
1010
"github.com/stretchr/testify/require"
1111

1212
"github.com/cortexproject/cortex/integration/e2e"
@@ -80,8 +80,9 @@ func TestAlertmanagerStoreAPI(t *testing.T) {
8080
err = c.SetAlertmanagerConfig(context.Background(), cortexAlertmanagerUserConfigYaml, map[string]string{})
8181
require.NoError(t, err)
8282

83-
time.Sleep(2 * time.Second)
84-
require.NoError(t, am.WaitSumMetrics(e2e.Equals(0), "cortex_alertmanager_config_invalid"))
83+
require.NoError(t, am.WaitSumMetricsWithOptions(e2e.Equals(0), []string{"cortex_alertmanager_config_invalid"},
84+
e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "user", "user-1")),
85+
e2e.WaitMissingMetrics))
8586

8687
cfg, err := c.GetAlertmanagerConfig(context.Background())
8788
require.NoError(t, err)
@@ -97,7 +98,10 @@ func TestAlertmanagerStoreAPI(t *testing.T) {
9798
err = c.DeleteAlertmanagerConfig(context.Background())
9899
require.NoError(t, err)
99100

100-
time.Sleep(2 * time.Second)
101+
// The deleted config is applied asynchronously, so we should wait until the metric
102+
// disappear for the specific user.
103+
require.NoError(t, am.WaitRemovedMetric("cortex_alertmanager_config_invalid", e2e.WithLabelMatchers(
104+
labels.MustNewMatcher(labels.MatchEqual, "user", "user-1"))))
101105

102106
cfg, err = c.GetAlertmanagerConfig(context.Background())
103107
require.Error(t, err)

‎integration/api_ruler_test.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"path/filepath"
77
"testing"
88

9+
"github.com/prometheus/prometheus/pkg/labels"
910
"github.com/prometheus/prometheus/pkg/rulefmt"
1011
"github.com/stretchr/testify/require"
1112
"gopkg.in/yaml.v3"
@@ -137,6 +138,9 @@ func TestRulerAPISingleBinary(t *testing.T) {
137138
require.Equal(t, retrievedNamespace[0].Name, "rule")
138139

139140
// Check to make sure prometheus engine metrics are available for both engine types
140-
require.NoError(t, cortex.WaitForMetricWithLabels(e2e.EqualsSingle(0), "prometheus_engine_queries", map[string]string{"engine": "querier"}))
141-
require.NoError(t, cortex.WaitForMetricWithLabels(e2e.EqualsSingle(0), "prometheus_engine_queries", map[string]string{"engine": "ruler"}))
141+
require.NoError(t, cortex.WaitSumMetricsWithOptions(e2e.Equals(0), []string{"prometheus_engine_queries"}, e2e.WithLabelMatchers(
142+
labels.MustNewMatcher(labels.MatchEqual, "engine", "querier"))))
143+
144+
require.NoError(t, cortex.WaitSumMetricsWithOptions(e2e.Equals(0), []string{"prometheus_engine_queries"}, e2e.WithLabelMatchers(
145+
labels.MustNewMatcher(labels.MatchEqual, "engine", "ruler"))))
142146
}

‎integration/e2e/composite_service.go

Lines changed: 16 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"fmt"
66
"time"
77

8+
"github.com/pkg/errors"
9+
810
"github.com/cortexproject/cortex/pkg/util"
911
)
1012

@@ -39,52 +41,41 @@ func (s *CompositeHTTPService) Instances() []*HTTPService {
3941
// WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true
4042
// when passed to given isExpected(...).
4143
func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error {
44+
return s.WaitSumMetricsWithOptions(isExpected, metricNames)
45+
}
46+
47+
func (s *CompositeHTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error {
4248
var (
43-
sums []float64
44-
err error
49+
sums []float64
50+
err error
51+
options = buildMetricsOptions(opts)
4552
)
4653

4754
for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
48-
sums, err = s.SumMetrics(metricNames...)
49-
if err != nil {
50-
return err
51-
}
52-
53-
if isExpected(sums...) {
54-
return nil
55+
sums, err = s.SumMetrics(metricNames, opts...)
56+
if options.WaitMissingMetrics && errors.Is(err, errMissingMetric) {
57+
continue
5558
}
56-
57-
s.retryBackoff.Wait()
58-
}
59-
60-
return fmt.Errorf("unable to find metrics %s with expected values. Last values: %v", metricNames, sums)
61-
}
62-
63-
func (s *CompositeHTTPService) WaitSumMetricWithLabels(isExpected func(sums float64) bool, metricName string, expectedLabels map[string]string) error {
64-
lastSum := 0.0
65-
66-
for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
67-
lastSum, err := s.SumMetricWithLabels(metricName, expectedLabels)
6859
if err != nil {
6960
return err
7061
}
7162

72-
if isExpected(lastSum) {
63+
if isExpected(sums...) {
7364
return nil
7465
}
7566

7667
s.retryBackoff.Wait()
7768
}
7869

79-
return fmt.Errorf("unable to find metric %s with labels %v with expected value. Last value: %v", metricName, expectedLabels, lastSum)
70+
return fmt.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums)
8071
}
8172

8273
// SumMetrics returns the sum of the values of each given metric names.
83-
func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
74+
func (s *CompositeHTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) {
8475
sums := make([]float64, len(metricNames))
8576

8677
for _, service := range s.services {
87-
partials, err := service.SumMetrics(metricNames...)
78+
partials, err := service.SumMetrics(metricNames, opts...)
8879
if err != nil {
8980
return nil, err
9081
}
@@ -100,19 +91,3 @@ func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, err
10091

10192
return sums, nil
10293
}
103-
104-
// SumMetricWithLabels returns the sum of the values of metric with matching labels across all services.
105-
func (s *CompositeHTTPService) SumMetricWithLabels(metricName string, expectedLabels map[string]string) (float64, error) {
106-
sum := 0.0
107-
108-
for _, service := range s.services {
109-
s, err := service.SumMetricWithLabels(metricName, expectedLabels)
110-
if err != nil {
111-
return 0, err
112-
}
113-
114-
sum += s
115-
}
116-
117-
return sum, nil
118-
}

‎integration/e2e/metrics.go

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
io_prometheus_client "github.com/prometheus/client_model/go"
77
)
88

9-
func getValue(m *io_prometheus_client.Metric) float64 {
9+
func getMetricValue(m *io_prometheus_client.Metric) float64 {
1010
if m.GetGauge() != nil {
1111
return m.GetGauge().GetValue()
1212
} else if m.GetCounter() != nil {
@@ -20,10 +20,63 @@ func getValue(m *io_prometheus_client.Metric) float64 {
2020
}
2121
}
2222

23-
func sumValues(family *io_prometheus_client.MetricFamily) float64 {
23+
func getMetricCount(m *io_prometheus_client.Metric) float64 {
24+
if m.GetHistogram() != nil {
25+
return float64(m.GetHistogram().GetSampleCount())
26+
} else if m.GetSummary() != nil {
27+
return float64(m.GetSummary().GetSampleCount())
28+
} else {
29+
return 0
30+
}
31+
}
32+
33+
func getValues(metrics []*io_prometheus_client.Metric, opts MetricsOptions) []float64 {
34+
values := make([]float64, 0, len(metrics))
35+
for _, m := range metrics {
36+
values = append(values, opts.GetValue(m))
37+
}
38+
return values
39+
}
40+
41+
func filterMetrics(metrics []*io_prometheus_client.Metric, opts MetricsOptions) []*io_prometheus_client.Metric {
42+
// If no label matcher is configured, then no filtering should be done.
43+
if len(opts.LabelMatchers) == 0 {
44+
return metrics
45+
}
46+
if len(metrics) == 0 {
47+
return metrics
48+
}
49+
50+
filtered := make([]*io_prometheus_client.Metric, 0, len(metrics))
51+
52+
for _, m := range metrics {
53+
metricLabels := map[string]string{}
54+
for _, lp := range m.GetLabel() {
55+
metricLabels[lp.GetName()] = lp.GetValue()
56+
}
57+
58+
matches := true
59+
for _, matcher := range opts.LabelMatchers {
60+
if !matcher.Matches(metricLabels[matcher.Name]) {
61+
matches = false
62+
break
63+
}
64+
}
65+
66+
if !matches {
67+
continue
68+
}
69+
70+
filtered = append(filtered, m)
71+
}
72+
73+
return filtered
74+
}
75+
76+
func sumValues(values []float64) float64 {
2477
sum := 0.0
25-
for _, m := range family.Metric {
26-
sum += getValue(m)
78+
for _, v := range values {
79+
sum += v
2780
}
2881
return sum
2982
}

‎integration/e2e/metrics_options.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package e2e
2+
3+
import (
4+
io_prometheus_client "github.com/prometheus/client_model/go"
5+
"github.com/prometheus/prometheus/pkg/labels"
6+
)
7+
8+
var (
9+
DefaultMetricsOptions = MetricsOptions{
10+
GetValue: getMetricValue,
11+
WaitMissingMetrics: false,
12+
}
13+
)
14+
15+
// GetMetricValueFunc defined the signature of a function used to get the metric value.
16+
type GetMetricValueFunc func(m *io_prometheus_client.Metric) float64
17+
18+
// MetricsOption defined the signature of a function used to manipulate options.
19+
type MetricsOption func(*MetricsOptions)
20+
21+
// MetricsOptions is the structure holding all options.
22+
type MetricsOptions struct {
23+
GetValue GetMetricValueFunc
24+
LabelMatchers []*labels.Matcher
25+
WaitMissingMetrics bool
26+
}
27+
28+
// WithMetricCount is an option to get the histogram/summary count as metric value.
29+
func WithMetricCount(opts *MetricsOptions) {
30+
opts.GetValue = getMetricCount
31+
}
32+
33+
// WithLabelMatchers is an option to filter only matching series.
34+
func WithLabelMatchers(matchers ...*labels.Matcher) MetricsOption {
35+
return func(opts *MetricsOptions) {
36+
opts.LabelMatchers = matchers
37+
}
38+
}
39+
40+
// WithWaitMissingMetrics is an option to wait whenever an expected metric is missing. If this
41+
// option is not enabled, will return error on missing metrics.
42+
func WaitMissingMetrics(opts *MetricsOptions) {
43+
opts.WaitMissingMetrics = true
44+
}
45+
46+
func buildMetricsOptions(opts []MetricsOption) MetricsOptions {
47+
result := DefaultMetricsOptions
48+
for _, opt := range opts {
49+
opt(&result)
50+
}
51+
return result
52+
}

0 commit comments

Comments
 (0)