feat: generate metrics from traces (#3453)

* [Feature] Generate metrics from traces #3389 * Minor fixes * Add name to new files --------- Co-authored-by: Ian Cooper <[email protected]>
BrighterCommand · Jan 3, 2025 · 071deb5 · 071deb5
1 parent d5522e6
commit 071deb5
Show file tree

Hide file tree

Showing 19 changed files with 781 additions and 9 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -67,6 +67,7 @@
     <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.10.0" />
     <PackageVersion Include="OpenTelemetry.Exporter.Jaeger" Version="1.5.1" />
     <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.10.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.10.0-beta.1" />
     <PackageVersion Include="OpenTelemetry.Exporter.Zipkin" Version="1.9.0" />
     <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.10.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.10.1" />

diff --git a/docker-compose-prometheus.yaml b/docker-compose-prometheus.yaml
@@ -7,3 +7,5 @@ services:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -113,6 +113,8 @@ services:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
 networks:
   kafka:

diff --git a/prometheus.yml b/prometheus.yml
@@ -5,3 +5,12 @@ scrape_configs:
   - job_name: 'otel-collector'
     static_configs:
       - targets: ['otel-collector:9090']
+
+  - job_name: 'brighter-sample-app'
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+    static_configs:
+      - targets:
+          - 'host.docker.internal:5001'
+    metrics_path: /metrics
diff --git a/samples/WebAPI/WebAPI_Dapper/GreetingsWeb/GreetingsWeb.csproj b/samples/WebAPI/WebAPI_Dapper/GreetingsWeb/GreetingsWeb.csproj
@@ -9,6 +9,7 @@
     <PackageReference Include="FluentMigrator.Runner"/>
     <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"/>
     <PackageReference Include="OpenTelemetry.Exporter.Console"/>
+    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting"/>
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore"/>
     <PackageReference Include="Paramore.Darker.AspNetCore"/>

diff --git a/samples/WebAPI/WebAPI_Dapper/GreetingsWeb/Startup.cs b/samples/WebAPI/WebAPI_Dapper/GreetingsWeb/Startup.cs
@@ -18,6 +18,7 @@
 using Paramore.Brighter;
 using Paramore.Brighter.Extensions.DependencyInjection;
 using Paramore.Brighter.Extensions.Diagnostics;
+using Paramore.Brighter.Observability;
 using Paramore.Darker.AspNetCore;
 using Paramore.Darker.Policies;
 using Paramore.Darker.QueryLogging;
@@ -47,6 +48,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         app.UseRouting();
 
         app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
+
+        app.UseOpenTelemetryPrometheusScrapingEndpoint();
     }
 
 
@@ -153,12 +156,13 @@ private void ConfigureObservability(IServiceCollection services)
                     serviceName: "GreetingsWeb",
                     serviceVersion: typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown",
                     serviceInstanceId: Environment.MachineName);
-            }).WithTracing(builder =>
+            })
+            .WithTracing(builder =>
             {
                 builder
                     .AddBrighterInstrumentation()
                     .AddSource("RabbitMQ.Client.*")
-                    .SetSampler(new AlwaysOnSampler())
+                    .SetTailSampler<AlwaysOnSampler>()
                     .AddAspNetCoreInstrumentation()
                     .AddConsoleExporter()
                     .AddOtlpExporter(options =>
@@ -169,7 +173,9 @@ private void ConfigureObservability(IServiceCollection services)
             .WithMetrics(builder => builder
                 .AddAspNetCoreInstrumentation()
                 .AddConsoleExporter()
+                .AddPrometheusExporter()
                 .AddOtlpExporter()
+                .AddBrighterInstrumentation()
             ); 
     }
 

diff --git a/samples/WebAPI/WebAPI_Dapper/SalutationAnalytics/Program.cs b/samples/WebAPI/WebAPI_Dapper/SalutationAnalytics/Program.cs
@@ -167,7 +167,7 @@ static void ConfigureObservability(IServiceCollection services)
             builder
                 .AddBrighterInstrumentation()
                 .AddSource("RabbitMQ.Client.*")
-                .SetSampler(new AlwaysOnSampler())
+                .SetTailSampler<AlwaysOffSampler>()
                 .AddAspNetCoreInstrumentation()
                 .AddConsoleExporter()
                 .AddOtlpExporter(options =>

diff --git a/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs b/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs
@@ -0,0 +1,46 @@
+#region Licence
+
+/* The MIT License (MIT)
+Copyright © 2025 Tim Salva <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE. */
+
+#endregion
+
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using OpenTelemetry.Metrics;
+using Paramore.Brighter.Observability;
+
+namespace Paramore.Brighter.Extensions.Diagnostics;
+
+public static class BrighterMetricsBuilderExtensions
+{
+    public static MeterProviderBuilder AddBrighterInstrumentation(this MeterProviderBuilder builder)
+    {
+        builder.ConfigureServices(services =>
+        {
+            services.TryAddSingleton<IAmABrighterMessagingMeter, MessagingMeter>();
+            services.TryAddSingleton<IAmABrighterDbMeter, DbMeter>();
+
+            builder.AddMeter(BrighterSemanticConventions.MeterName);
+        });
+
+        return builder;
+    }
+}
diff --git a/src/Paramore.Brighter.Extensions.Diagnostics/BrighterTracerBuilderExtensions.cs b/src/Paramore.Brighter.Extensions.Diagnostics/BrighterTracerBuilderExtensions.cs
@@ -17,9 +17,33 @@ public static TracerProviderBuilder AddBrighterInstrumentation(this TracerProvid
 
 
             builder.AddSource(brighterTracer.ActivitySource.Name);
+            builder.AddProcessor<BrighterMetricsFromTracesProcessor>();
         });
 
         return builder;
     }
-
+
+    /// <summary>
+    /// Register a tail-based sampler delegating the sampling decision to any OpenTelemetry Sampler.
+    /// </summary>
+    /// <remarks>
+    /// Examples:
+    /// <code>
+    /// // Use a TraceIdRatioBasedSampler with 25% sampling rate
+    /// builder.SetTailSampler(new TraceIdRatioBasedSampler(0.25));
+    /// 
+    /// // Use default constructor of AlwaysOnSampler
+    /// builder.SetTailSampler&lt;AlwaysOnSampler&gt;();
+    /// </code>
+    /// </remarks>
+    /// <param name="builder">The TracerProviderBuilder to configure</param>
+    /// <param name="sampler">Optional sampler instance. If null, will create new instance using default constructor</param>
+    /// <typeparam name="TSampler">Type of sampler to use</typeparam>
+    /// <returns>The TracerProviderBuilder for chaining</returns>
+    public static TracerProviderBuilder SetTailSampler<TSampler>(this TracerProviderBuilder builder, TSampler sampler = null)
+        where TSampler : Sampler, new()
+    {
+        builder.AddProcessor(new TailSamplerProcessor(sampler ?? new TSampler()));
+        return builder;
+    }
 }
diff --git a/src/Paramore.Brighter/Observability/BrighterMetricsFromTracesProcessor.cs b/src/Paramore.Brighter/Observability/BrighterMetricsFromTracesProcessor.cs
@@ -0,0 +1,86 @@
+#region Licence
+
+/* The MIT License (MIT)
+Copyright © 2025 Tim Salva <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE. */
+
+#endregion
+
+using System.Diagnostics;
+using OpenTelemetry;
+
+namespace Paramore.Brighter.Observability;
+
+/// <summary>
+/// Generates metrics from traces following OpenTelemetry Semantic Conventions 1.29.0
+/// Unstable due to depending on experimental parts of the OpenTelemetry spec
+/// </summary>
+public sealed class BrighterMetricsFromTracesProcessor(
+    IAmABrighterTracer brighterTracer,
+    IAmABrighterDbMeter dbMeter,
+    IAmABrighterMessagingMeter messagingMeter)
+    : BaseProcessor<Activity>
+{
+    private readonly string _brighterActivitySourceName = brighterTracer.ActivitySource.Name;
+
+    private bool Enabled => dbMeter.Enabled || messagingMeter.Enabled;
+
+    public override void OnEnd(Activity? activity)
+    {
+        if (!Enabled) return;
+
+        if (activity == null) return;
+
+        if (!activity.Source.Name.Equals(_brighterActivitySourceName)) return;
+
+        if (activity.GetTagItem(BrighterSemanticConventions.InstrumentationDomain) is not string instrumentationDomain) return;
+
+        switch (instrumentationDomain)
+        {
+            case BrighterSemanticConventions.MessagingInstrumentationDomain:
+                if (activity.GetTagItem(BrighterSemanticConventions.MessagingOperationType) is string operation)
+                {
+                    switch (operation)
+                    {
+                        case "publish":
+                            messagingMeter.RecordClientOperation(activity);
+                            messagingMeter.AddClientSentMessage(activity);
+                            break;
+                        case "receive":
+                            messagingMeter.RecordClientOperation(activity);
+                            messagingMeter.AddClientConsumedMessage(activity);
+                            break;
+                        case "process":
+                            messagingMeter.RecordProcess(activity);
+                            break;
+                        default:
+                            messagingMeter.RecordClientOperation(activity);
+                            break;
+                    }
+                }
+                break;
+            case BrighterSemanticConventions.DbInstrumentationDomain:
+                dbMeter.RecordClientOperation(activity);
+                break;
+        }
+
+        base.OnEnd(activity);
+    }
+}
diff --git a/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs b/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs
@@ -35,10 +35,18 @@ public static class BrighterSemanticConventions
     public const string CeMessageId = "cloudevents.event_id";
     public const string CeVersion = "cloudevents.event_spec_version";
     public const string ClearMessages = "paramore.brighter.clear_messages";
+    public const string ConsumerGroupName = "messaging.consumer.group.name";
     public const string ConversationId = "messaging.message.conversation_id";
+    public const string DbCollectionName = "db.collection.name";
     public const string DbInstanceId = "db.instance.id";
+    public const string DbInstrumentationDomain = "db";
     public const string DbName = "db.name";
+    public const string DbNamespace = "db.namespace";
     public const string DbOperation = "db.operation";
+    public const string DbOperationName = "db.operation.name";
+    public const string DbQuerySummary = "db.query.summary";
+    public const string DbQueryText = "db.query.text";
+    public const string DbResponseStatusCode = "db.response.status_code";
     public const string DbStatement = "db.statement";
     public const string DbSystem = "db.system";
     public const string DbTable = "db.table";
@@ -47,6 +55,7 @@ public static class BrighterSemanticConventions
     public const string HandledCount = "paramore.brighter.handled_count";
     public const string HandlerName = "paramore.brighter.handler.name";
     public const string HandlerType = "paramore.brighter.handler.type";
+    public const string InstrumentationDomain = "instrumentation.domain";
     public const string IsSink = "paramore.brighter.is_sink";
     public const string MapperName = "paramore.brighter.mapper.name";
     public const string MapperType = "paramore.brighter.mapper.type";
@@ -57,9 +66,14 @@ public static class BrighterSemanticConventions
     public const string MessagingDestination = "messaging.destination.name";
     public const string MessagingDestinationAnonymous = "messaging.destination.anonymous";
     public const string MessagingDestinationPartitionId = "messaging.destination.partition.id";
+    public const string MessagingDestinationSubscriptionName = "messaging.destination.subscription.name";
+    public const string MessagingDestinationTemplate = "messaging.destination.template";
+    public const string MessagingInstrumentationDomain = "messaging";
+    public const string MessagingOperationName = "messaging.operation.name";
     public const string MessagingOperationType = "messaging.operation.type";
     public const string MessagingSystem = "messaging.system";
     public const string MessageBodySize = "messaging.message.body.size";
+    public const string MeterName = "Paramore.Brighter";
     public const string NetworkPeerAddress = "network.peer.address";
     public const string NetworkPeerPort = "network.peer.port";
     public const string Operation = "paramore.brighter.operation";
@@ -70,7 +84,11 @@ public static class BrighterSemanticConventions
     public const string RequestType = "paramore.brighter.request.type"; 
     public const string RequestBody = "paramore.brighter.request.body";
     public const string ServerAddress = "server.address";
-    public const string ServerPort ="server.port";
+    public const string ServerPort = "server.port";
+    public const string ServiceInstanceId = "service.instance.id";
+    public const string ServiceName = "service.name";
+    public const string ServiceNamespace = "service.namespace";
+    public const string ServiceVersion = "service.version";
     public const string SourceName = "Paramore.Brighter";
     public const string CeSubject = "cloudevents.event_subject";
     public const string CeType = "cloudevents.event_type";