diff --git a/exporter/prometheusexporter/example_test.go b/exporter/prometheusexporter/example_test.go new file mode 100644 index 00000000..b2d34d1d --- /dev/null +++ b/exporter/prometheusexporter/example_test.go @@ -0,0 +1,108 @@ +// Copyright 2019, OpenCensus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter_test + +import ( + "context" + "log" + "net/http" + "time" + + "github.com/census-instrumentation/opencensus-service/exporter/prometheusexporter" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/golang/protobuf/ptypes/timestamp" +) + +func Example() { + pe, err := prometheusexporter.New(prometheusexporter.Options{}) + if err != nil { + log.Fatalf("Failed to create new exporter: %v", err) + } + + mux := http.NewServeMux() + // Expose the Prometheus Metrics exporter for scraping on route /metrics. + mux.Handle("/metrics", pe) + + // Now run the server. + go func() { + http.ListenAndServe(":8888", mux) + }() + + // And finally in your client application, use the + // OpenCensus-Go Metrics Prometheus exporter to record metrics. + for { + pe.ExportMetric(context.Background(), nil, nil, metric1) + // Introducing a fake pause/period. + <-time.After(350 * time.Millisecond) + } +} + +var ( + startTimestamp = ×tamp.Timestamp{ + Seconds: 1543160298, + Nanos: 100000090, + } + endTimestamp = ×tamp.Timestamp{ + Seconds: 1543160298, + Nanos: 100000997, + } +) + +var metric1 = &metricspb.Metric{ + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "this/one/there(where)", + Description: "Extra ones", + Unit: "1", + LabelKeys: []*metricspb.LabelKey{ + {Key: "os", Description: "Operating system"}, + {Key: "arch", Description: "Architecture"}, + }, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + LabelValues: []*metricspb.LabelValue{ + {Value: "windows"}, + {Value: "x86"}, + }, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_Int64Value{ + Int64Value: 99, + }, + }, + }, + }, + { + StartTimestamp: startTimestamp, + LabelValues: []*metricspb.LabelValue{ + {Value: "darwin"}, + {Value: "386"}, + }, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 49.5, + }, + }, + }, + }, + }, +} diff --git a/exporter/prometheusexporter/exporter.go b/exporter/prometheusexporter/exporter.go new file mode 100644 index 00000000..69878f1f --- /dev/null +++ b/exporter/prometheusexporter/exporter.go @@ -0,0 +1,352 @@ +// Copyright 2019, OpenCensus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "net/http" + "sort" + "sync" + + "go.opencensus.io/trace" + + commonpb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/common/v1" + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Exporter is the type that converts OpenCensus Proto-Metrics into Prometheus metrics. +type Exporter struct { + options Options + handler http.Handler + collector *collector + gatherer prometheus.Gatherer +} + +// Options customizes a created Prometheus Exporter. +type Options struct { + Namespace string + OnError func(err error) + ConstLabels prometheus.Labels // ConstLabels will be set as labels on all views. + Registry *prometheus.Registry +} + +// New is the constructor to make an Exporter with the defined Options. +func New(o Options) (*Exporter, error) { + // It is imperative that Options.Registry is non-nil when the + // Prometheus client uses it, but also that when creating + // the "collector" below, we use a non-nil Registry. + if o.Registry == nil { + o.Registry = prometheus.NewRegistry() + } + collector := newCollector(o, o.Registry) + exp := &Exporter{ + options: o, + gatherer: o.Registry, + collector: collector, + handler: promhttp.HandlerFor(o.Registry, promhttp.HandlerOpts{}), + } + return exp, nil +} + +type collector struct { + mu sync.Mutex + opts Options + registry *prometheus.Registry + registerOnce sync.Once + registeredMetricsMu sync.Mutex + registeredMetrics map[string]*prometheus.Desc + metricsData map[string]*metricspb.Metric +} + +func newCollector(opts Options, registry *prometheus.Registry) *collector { + return &collector{ + registry: registry, + opts: opts, + registeredMetrics: make(map[string]*prometheus.Desc), + metricsData: make(map[string]*metricspb.Metric), + } +} + +var _ http.Handler = (*Exporter)(nil) + +func (c *collector) lookupPrometheusDesc(namespace string, metric *metricspb.Metric) (*prometheus.Desc, string, bool) { + signature := metricSignature(namespace, metric) + c.registeredMetricsMu.Lock() + desc, ok := c.registeredMetrics[signature] + c.registeredMetricsMu.Unlock() + + return desc, signature, ok +} + +func (c *collector) registerMetrics(metrics ...*metricspb.Metric) error { + count := 0 + for _, metric := range metrics { + _, signature, ok := c.lookupPrometheusDesc(c.opts.Namespace, metric) + + if !ok { + desc := prometheus.NewDesc( + metricName(c.opts.Namespace, metric), + metric.GetMetricDescriptor().GetDescription(), + protoLabelKeysToLabels(metric.GetMetricDescriptor().GetLabelKeys()), + c.opts.ConstLabels, + ) + c.registeredMetricsMu.Lock() + c.registeredMetrics[signature] = desc + c.registeredMetricsMu.Unlock() + count++ + } + } + + if count == 0 { + return nil + } + + return c.ensureRegisteredOnce() +} + +func metricName(namespace string, metric *metricspb.Metric) string { + var name string + if namespace != "" { + name = namespace + "_" + } + mName := metric.GetMetricDescriptor().GetName() + if mName == "" { + mName = metric.GetName() + } + return name + sanitize(mName) +} + +func metricSignature(namespace string, metric *metricspb.Metric) string { + var buf bytes.Buffer + buf.WriteString(metricName(namespace, metric)) + labelKeys := metric.GetMetricDescriptor().GetLabelKeys() + for _, labelKey := range labelKeys { + buf.WriteString("-" + labelKey.Key) + } + return buf.String() +} + +func (c *collector) ensureRegisteredOnce() error { + var finalErr error + c.registerOnce.Do(func() { + if err := c.registry.Register(c); err != nil { + finalErr = err + } + }) + return finalErr +} + +func (exp *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + exp.handler.ServeHTTP(w, r) +} + +func (o *Options) onError(err error) { + if o != nil && o.OnError != nil { + o.OnError(err) + } else { + log.Printf("Failed to export to Prometheus: %v", err) + } +} + +// ExportMetric is the method that the exporter uses to convert OpenCensus Proto-Metrics to Prometheus metrics. +func (exp *Exporter) ExportMetric(ctx context.Context, node *commonpb.Node, rsc *resourcepb.Resource, metric *metricspb.Metric) error { + if metric == nil || len(metric.Timeseries) == 0 { + return nil + } + + // TODO: (@odeke-em) use node, resource and metrics e.g. perhaps to derive default labels + return exp.collector.addMetric(metric) +} + +func (c *collector) addMetric(metric *metricspb.Metric) error { + if err := c.registerMetrics(metric); err != nil { + return err + } + + signature := metricSignature(c.opts.Namespace, metric) + + c.mu.Lock() + c.metricsData[signature] = metric + c.mu.Unlock() + + return nil +} + +func (c *collector) Describe(ch chan<- *prometheus.Desc) { + c.registeredMetricsMu.Lock() + registered := make(map[string]*prometheus.Desc) + for key, desc := range c.registeredMetrics { + registered[key] = desc + } + c.registeredMetricsMu.Unlock() + + for _, desc := range registered { + ch <- desc + } +} + +func (c *collector) Collect(ch chan<- prometheus.Metric) { + // We need a copy of all the metric data up until this point in time. + metricsData := c.cloneMetricsData() + + ctx, span := trace.StartSpan(context.Background(), "prometheus.Metrics.Collect", trace.WithSampler(trace.NeverSample())) + defer span.End() + + for _, metric := range metricsData { + for _, timeseries := range metric.Timeseries { + pmetrics, err := c.protoTimeSeriesToPrometheusMetrics(ctx, metric, timeseries) + if err == nil { + for _, pmetric := range pmetrics { + ch <- pmetric + } + } else { + c.opts.onError(err) + } + } + } +} + +var errNilTimeSeries = errors.New("expecting a non-nil TimeSeries") + +func (c *collector) protoTimeSeriesToPrometheusMetrics(ctx context.Context, metric *metricspb.Metric, ts *metricspb.TimeSeries) ([]prometheus.Metric, error) { + if ts == nil { + return nil, errNilTimeSeries + } + + labelKeys := metric.GetMetricDescriptor().GetLabelKeys() + labelValues, err := protoLabelValuesToLabelValues(labelKeys, ts.LabelValues) + if err != nil { + return nil, err + } + derivedPrometheusValueType := prometheusValueType(metric) + desc, _, _ := c.lookupPrometheusDesc(c.opts.Namespace, metric) + + pmetrics := make([]prometheus.Metric, 0, len(ts.Points)) + for _, point := range ts.Points { + pmet, err := protoMetricToPrometheusMetric(ctx, point, desc, derivedPrometheusValueType, labelValues) + if err == nil { + pmetrics = append(pmetrics, pmet) + } else { + // TODO: (@odeke-em) record these errors + } + } + return pmetrics, nil +} + +func protoLabelValuesToLabelValues(rubricLabelKeys []*metricspb.LabelKey, protoLabelValues []*metricspb.LabelValue) ([]string, error) { + if len(protoLabelValues) == 0 { + return nil, nil + } + + if len(protoLabelValues) > len(rubricLabelKeys) { + return nil, fmt.Errorf("len(LabelValues)=%d > len(labelKeys)=%d", len(protoLabelValues), len(rubricLabelKeys)) + } + plainLabelValues := make([]string, len(rubricLabelKeys)) + for i, protoLabelValue := range protoLabelValues { + if protoLabelValue.Value != "" || protoLabelValue.HasValue { + plainLabelValues[i] = protoLabelValue.Value + } + } + return plainLabelValues, nil +} + +func protoLabelKeysToLabels(protoLabelKeys []*metricspb.LabelKey) []string { + if len(protoLabelKeys) == 0 { + return nil + } + labelKeys := make([]string, 0, len(protoLabelKeys)) + for _, protoLabelKey := range protoLabelKeys { + sanitizedKey := sanitize(protoLabelKey.GetKey()) + labelKeys = append(labelKeys, sanitizedKey) + } + return labelKeys +} + +func protoMetricToPrometheusMetric(ctx context.Context, point *metricspb.Point, desc *prometheus.Desc, derivedPrometheusType prometheus.ValueType, labelValues []string) (prometheus.Metric, error) { + switch value := point.Value.(type) { + case *metricspb.Point_DistributionValue: + dValue := value.DistributionValue + + // Histograms are cumulative in Prometheus. + dBuckets := dValue.BucketOptions.GetExplicit().GetBounds() + buckets := make([]float64, 0, len(dValue.Buckets)) + indicesMap := make(map[float64]int, len(dBuckets)) + for index, bucket := range dBuckets { + if _, added := indicesMap[bucket]; !added { + indicesMap[bucket] = index + buckets = append(buckets, bucket) + } + } + sort.Float64s(buckets) + + // 2. Now that the buckets are sorted by magnitude, we can create + // cumulative indices and map them back by reverse index. + cumCount := uint64(0) + + points := make(map[float64]uint64, len(buckets)) + for _, bucket := range buckets { + index := indicesMap[bucket] + var countPerBucket uint64 + if len(dValue.Buckets) > 0 && index < len(dValue.Buckets) { + countPerBucket = uint64(dValue.Buckets[index].GetCount()) + } + cumCount += countPerBucket + points[bucket] = cumCount + } + return prometheus.NewConstHistogram(desc, uint64(dValue.Count), dValue.Sum, points, labelValues...) + + case *metricspb.Point_Int64Value: + // Derive the Prometheus + return prometheus.NewConstMetric(desc, prometheus.CounterValue, float64(value.Int64Value), labelValues...) + + case *metricspb.Point_DoubleValue: + return prometheus.NewConstMetric(desc, prometheus.CounterValue, value.DoubleValue, labelValues...) + + default: + return nil, fmt.Errorf("Unhandled type: %T", point.Value) + } +} + +func prometheusValueType(metric *metricspb.Metric) prometheus.ValueType { + switch metric.GetMetricDescriptor().GetType() { + case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_INT64, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: + return prometheus.GaugeValue + + case metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, metricspb.MetricDescriptor_CUMULATIVE_INT64, metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: + return prometheus.CounterValue + + default: + return prometheus.UntypedValue + } +} + +func (c *collector) cloneMetricsData() map[string]*metricspb.Metric { + c.mu.Lock() + defer c.mu.Unlock() + + metricsDataCopy := make(map[string]*metricspb.Metric, len(c.metricsData)) + for signature, metric := range c.metricsData { + metricsDataCopy[signature] = metric + } + return metricsDataCopy +} diff --git a/exporter/prometheusexporter/exporter_test.go b/exporter/prometheusexporter/exporter_test.go new file mode 100644 index 00000000..00f71a0d --- /dev/null +++ b/exporter/prometheusexporter/exporter_test.go @@ -0,0 +1,418 @@ +// Copyright 2019, OpenCensus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "context" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/golang/protobuf/ptypes/timestamp" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + startTimestamp = ×tamp.Timestamp{ + Seconds: 1543160298, + Nanos: 100000090, + } + endTimestamp = ×tamp.Timestamp{ + Seconds: 1543160298, + Nanos: 100000997, + } +) + +func TestOnlyCumulativeWindowSupported(t *testing.T) { + + tests := []struct { + metric *metricspb.Metric + wantCount int + }{ + { + metric: &metricspb.Metric{}, wantCount: 0, + }, + { + metric: &metricspb.Metric{ + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "with_metric_descriptor", + Description: "This is a test", + Unit: "By", + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Count: 1, + Sum: 11.9, + SumOfSquaredDeviation: 0, + Buckets: []*metricspb.DistributionValue_Bucket{ + {}, {Count: 1}, {}, {}, {}, + }, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10, 20, 30, 40}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantCount: 1, + }, + { + metric: &metricspb.Metric{ + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter", + Description: "This is a counter", + Unit: "1", + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_Int64Value{Int64Value: 197}, + }, + }, + }, + }, + }, + wantCount: 1, + }, + } + + for i, tt := range tests { + reg := prometheus.NewRegistry() + collector := newCollector(Options{}, reg) + collector.addMetric(tt.metric) + mm, err := reg.Gather() + if err != nil { + t.Errorf("#%d: Gather error: %v", i, err) + } + reg.Unregister(collector) + if got, want := len(mm), tt.wantCount; got != want { + t.Errorf("#%d: Got %d Want %d", i, got, want) + } + } +} + +func TestCollectNonRacy(t *testing.T) { + exp, err := New(Options{}) + if err != nil { + t.Fatalf("NewExporter failed: %v", err) + } + collector := exp.collector + + // Synchronization to ensure that every goroutine terminates before we exit. + var waiter sync.WaitGroup + waiter.Add(3) + defer waiter.Wait() + + doneCh := make(chan bool) + + // 1. Simulate metrics write route with a period of 700ns. + go func() { + defer waiter.Done() + tick := time.NewTicker(700 * time.Nanosecond) + + defer func() { + tick.Stop() + close(doneCh) + }() + + for i := 0; i < 1e3; i++ { + metrics := []*metricspb.Metric{ + { + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "with_metric_descriptor", + Description: "This is a test", + Unit: "By", + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Count: int64(i + 10), + Sum: 11.9 + float64(i), + SumOfSquaredDeviation: 0, + Buckets: []*metricspb.DistributionValue_Bucket{ + {}, {Count: 1}, {}, {}, {}, + }, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10, 20, 30, 40}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter", + Description: "This is a counter", + Unit: "1", + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_Int64Value{Int64Value: int64(i)}, + }, + }, + }, + }, + }, + } + + for _, metric := range metrics { + if err := exp.ExportMetric(context.Background(), nil, nil, metric); err != nil { + t.Errorf("Iteration #%d:: unexpected ExportMetric error: %v", i, err) + } + <-tick.C + } + } + }() + + inMetricsChan := make(chan prometheus.Metric, 1000) + // 2. Simulate the Prometheus metrics consumption routine running at 900ns. + go func() { + defer waiter.Done() + tick := time.NewTicker(900 * time.Nanosecond) + defer tick.Stop() + + for { + select { + case <-doneCh: + return + + case <-inMetricsChan: + case <-tick.C: + } + } + }() + + // 3. Collect/Read routine running at 800ns. + go func() { + defer waiter.Done() + tick := time.NewTicker(800 * time.Nanosecond) + defer tick.Stop() + + for { + select { + case <-doneCh: + return + + case <-tick.C: + // Perform some collection here. + collector.Collect(inMetricsChan) + } + } + }() +} + +func makeMetrics() []*metricspb.Metric { + return []*metricspb.Metric{ + { + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "with/metric*descriptor", + Description: "This is a test", + Unit: "By", + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Count: 2, + Sum: 61.9, + SumOfSquaredDeviation: 0, + Buckets: []*metricspb.DistributionValue_Bucket{ + {}, {Count: 1}, {}, {}, {Count: 5}, + }, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10, 20, 30, 40}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + Descriptor_: &metricspb.Metric_MetricDescriptor{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "this/one/there(where)", + Description: "Extra ones", + Unit: "1", + LabelKeys: []*metricspb.LabelKey{ + {Key: "os", Description: "Operating system"}, + {Key: "arch", Description: "Architecture"}, + {Key: "my.org/department", Description: "The department that owns this server"}, + }, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: startTimestamp, + LabelValues: []*metricspb.LabelValue{ + {Value: "windows"}, + {Value: "x86"}, + {Value: "Storage"}, + }, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_Int64Value{ + Int64Value: 99, + }, + }, + }, + }, + { + StartTimestamp: startTimestamp, + LabelValues: []*metricspb.LabelValue{ + {Value: "darwin"}, + {Value: "386"}, + {Value: "Ops"}, + }, + Points: []*metricspb.Point{ + { + Timestamp: endTimestamp, + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 49.5, + }, + }, + }, + }, + }, + }, + } +} + +func TestMetricsEndpointOutput(t *testing.T) { + exp, err := New(Options{}) + if err != nil { + t.Fatalf("Failed to create Prometheus exporter: %v", err) + } + + srv := httptest.NewServer(exp) + defer srv.Close() + + // Now record some metrics. + metrics := makeMetrics() + for _, metric := range metrics { + exp.ExportMetric(context.Background(), nil, nil, metric) + } + + var i int + var output string + for { + time.Sleep(10 * time.Millisecond) + if i == 1000 { + t.Fatal("no output at / (10s wait)") + } + i++ + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("Failed to get metrics on / error: %v", err) + } + + slurp, err := ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Fatalf("Failed to read body: %v", err) + } + + output = string(slurp) + if output != "" { + break + } + } + + if strings.Contains(output, "collected before with the same name and label values") { + t.Fatalf("metric name and labels were duplicated but must be unique. Got\n\t%q", output) + } + + if strings.Contains(output, "error(s) occurred") { + t.Fatalf("error reported by Prometheus registry:\n\t%s", output) + } + + want := `# HELP this_one_there_where_ Extra ones +# TYPE this_one_there_where_ counter +this_one_there_where_{arch="386",my_org_department="Ops",os="darwin"} 49.5 +this_one_there_where_{arch="x86",my_org_department="Storage",os="windows"} 99 +# HELP with_metric_descriptor This is a test +# TYPE with_metric_descriptor histogram +with_metric_descriptor_bucket{le="0"} 0 +with_metric_descriptor_bucket{le="10"} 1 +with_metric_descriptor_bucket{le="20"} 1 +with_metric_descriptor_bucket{le="30"} 1 +with_metric_descriptor_bucket{le="40"} 6 +with_metric_descriptor_bucket{le="+Inf"} 2 +with_metric_descriptor_sum 61.9 +with_metric_descriptor_count 2 +` + if g, w := output, want; g != w { + t.Errorf("Mismatched output\nGot:\n%s\nWant:\n%s", g, w) + } +} diff --git a/exporter/prometheusexporter/prometheus.go b/exporter/prometheusexporter/prometheus.go index 9f2d505f..601cdb70 100644 --- a/exporter/prometheusexporter/prometheus.go +++ b/exporter/prometheusexporter/prometheus.go @@ -25,10 +25,6 @@ import ( "github.com/census-instrumentation/opencensus-service/data" "github.com/spf13/viper" - // TODO: once this repository has been transferred to the - // official census-ecosystem location, update this import path. - "github.com/orijtech/prometheus-go-metrics-exporter" - prometheus_golang "github.com/prometheus/client_golang/prometheus" ) @@ -66,11 +62,11 @@ func PrometheusExportersFromViper(v *viper.Viper) (tps []consumer.TraceConsumer, return } - opts := prometheus.Options{ + opts := Options{ Namespace: pcfg.Namespace, ConstLabels: pcfg.ConstLabels, } - pe, err := prometheus.New(opts) + pe, err := New(opts) if err != nil { return nil, nil, nil, err } @@ -98,7 +94,7 @@ func PrometheusExportersFromViper(v *viper.Viper) (tps []consumer.TraceConsumer, } type prometheusExporter struct { - exporter *prometheus.Exporter + exporter *Exporter } var _ consumer.MetricsConsumer = (*prometheusExporter)(nil) diff --git a/exporter/prometheusexporter/sanitize.go b/exporter/prometheusexporter/sanitize.go new file mode 100644 index 00000000..73e8c732 --- /dev/null +++ b/exporter/prometheusexporter/sanitize.go @@ -0,0 +1,53 @@ +// Copyright 2019, OpenCensus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +// The code for sanitize is copied verbatim from: +// https://github.com/census-instrumentation/opencensus-go/blob/950a67f393d867cfbe91414063b69e511f42fefb/internal/sanitize.go#L1-L50 + +import ( + "strings" + "unicode" +) + +const labelKeySizeLimit = 100 + +// sanitize returns a string that is trunacated to 100 characters if it's too +// long, and replaces non-alphanumeric characters to underscores. +func sanitize(s string) string { + if len(s) == 0 { + return s + } + if len(s) > labelKeySizeLimit { + s = s[:labelKeySizeLimit] + } + s = strings.Map(sanitizeRune, s) + if unicode.IsDigit(rune(s[0])) { + s = "key_" + s + } + if s[0] == '_' { + s = "key" + s + } + return s +} + +// converts anything that is not a letter or digit to an underscore +func sanitizeRune(r rune) rune { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + return r + } + // Everything else turns into an underscore + return '_' +} diff --git a/exporter/prometheusexporter/sanitize_test.go b/exporter/prometheusexporter/sanitize_test.go new file mode 100644 index 00000000..86040edd --- /dev/null +++ b/exporter/prometheusexporter/sanitize_test.go @@ -0,0 +1,67 @@ +// Copyright 2017, OpenCensus Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "strings" + "testing" +) + +func TestSanitize(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "truncate long string", + input: strings.Repeat("a", 101), + want: strings.Repeat("a", 100), + }, + { + name: "replace character", + input: "test/key-1", + want: "test_key_1", + }, + { + name: "add prefix if starting with digit", + input: "0123456789", + want: "key_0123456789", + }, + { + name: "add prefix if starting with _", + input: "_0123456789", + want: "key_0123456789", + }, + { + name: "starts with _ after sanitization", + input: "/0123456789", + want: "key_0123456789", + }, + { + name: "valid input", + input: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0123456789", + want: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0123456789", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, want := sanitize(tt.input), tt.want; got != want { + t.Errorf("sanitize() = %q; want %q", got, want) + } + }) + } +} diff --git a/go.mod b/go.mod index 4d5f1462..37bf2a1a 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( github.com/onsi/gomega v1.4.3 // indirect github.com/opentracing/opentracing-go v1.0.2 // indirect github.com/openzipkin/zipkin-go v0.1.3 - github.com/orijtech/prometheus-go-metrics-exporter v0.0.2 github.com/orijtech/promreceiver v0.0.3 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/philhofer/fwd v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6a94a9d2..7792cd3c 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ cloud.google.com/go v0.32.0 h1:DSt59WoyNcfAInilEpfvm2ugq8zvNyaHAm9MkzOwRQ4= cloud.google.com/go v0.32.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0 h1:YsbWYxDZkC7x2OxlsDEYvvEXZ3cBI3qBgUK5BqkZvRw= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/ocagent v0.4.4 h1:6v7OlUmiBDhvbYcHPWp8LHO8wh9jJY8f4mO34J4GJiA= -contrib.go.opencensus.io/exporter/ocagent v0.4.4/go.mod h1:YuG83h+XWwqWjvCqn7vK4KSyLKhThY3+gNGQ37iS2V0= contrib.go.opencensus.io/exporter/ocagent v0.4.6 h1:xVeoJwnzMbseoL9YWhohR6SN/GncvP1p/fznasLkT/E= contrib.go.opencensus.io/exporter/ocagent v0.4.6/go.mod h1:YuG83h+XWwqWjvCqn7vK4KSyLKhThY3+gNGQ37iS2V0= contrib.go.opencensus.io/exporter/stackdriver v0.9.1 h1:W6APgQ9we4BH8U8bnq/FvwLKo2WSMHuiMkkS/Slkg30= @@ -296,8 +294,6 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.3 h1:36hTtUTQR/vPX7YVJo2PYexSbHdAJiAkDrjuXw/YlYQ= github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/orijtech/prometheus-go-metrics-exporter v0.0.2 h1:XZGTMsJ8xM4VyrFGwHkdpZNnexxmAzFaq1//gUWqyKE= -github.com/orijtech/prometheus-go-metrics-exporter v0.0.2/go.mod h1:+Mu9w51Uc2RNKSUTA95d6Pvy8cxFiRX3ANRPlCcnGLA= github.com/orijtech/promreceiver v0.0.3 h1:qK3QOv1JdLtD+8Mc0bx3RdDPTVE1uUBI25YtttuWATc= github.com/orijtech/promreceiver v0.0.3/go.mod h1:2MEABETYIwm/Don8X1s/mv8R5FwJbtwo4GwxW2qAFYQ= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=