diff --git a/src/Observability/Extensions/OpenAI/BuilderExtensions.cs b/src/Observability/Extensions/OpenAI/BuilderExtensions.cs index a0105ebc..78b9ed8a 100644 --- a/src/Observability/Extensions/OpenAI/BuilderExtensions.cs +++ b/src/Observability/Extensions/OpenAI/BuilderExtensions.cs @@ -17,8 +17,9 @@ public static class BuilderExtensions /// /// The builder to configure. /// Whether to enable related tracing sources for OpenTelemetry. + /// Configuration options for OpenAI span processing. If null, default options will be used. /// The configured builder for method chaining. - public static Builder WithOpenAI(this Builder builder, bool enableRelatedSources = true) + public static Builder WithOpenAI(this Builder builder, bool enableRelatedSources = true, OpenAISpanProcessorOptions? options = null) { if (enableRelatedSources) { @@ -26,7 +27,7 @@ public static Builder WithOpenAI(this Builder builder, bool enableRelatedSources builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddSource(OpenAITelemetryConstants.OpenAISourceWildcard) - .AddProcessor(new OpenAISpanProcessor())); + .AddProcessor(new OpenAISpanProcessor(options ?? new OpenAISpanProcessorOptions()))); } return builder; diff --git a/src/Observability/Extensions/OpenAI/Microsoft.Agents.A365.Observability.Extensions.OpenAI.csproj b/src/Observability/Extensions/OpenAI/Microsoft.Agents.A365.Observability.Extensions.OpenAI.csproj index 29b65992..57a9f37f 100644 --- a/src/Observability/Extensions/OpenAI/Microsoft.Agents.A365.Observability.Extensions.OpenAI.csproj +++ b/src/Observability/Extensions/OpenAI/Microsoft.Agents.A365.Observability.Extensions.OpenAI.csproj @@ -18,6 +18,9 @@ + + + diff --git a/src/Observability/Extensions/OpenAI/OpenAISpanProcessor.cs b/src/Observability/Extensions/OpenAI/OpenAISpanProcessor.cs index 13245a46..629a776a 100644 --- a/src/Observability/Extensions/OpenAI/OpenAISpanProcessor.cs +++ b/src/Observability/Extensions/OpenAI/OpenAISpanProcessor.cs @@ -11,6 +11,12 @@ namespace Microsoft.Agents.A365.Observability.Extensions.OpenAI; internal class OpenAISpanProcessor : BaseProcessor { private static readonly string TargetSourceName = OpenAITelemetryConstants.OpenAISource; + private readonly OpenAISpanProcessorOptions _options; + + public OpenAISpanProcessor(OpenAISpanProcessorOptions options) + { + _options = options ?? new OpenAISpanProcessorOptions(); + } public override void OnStart(Activity activity) { @@ -32,5 +38,19 @@ public override void OnEnd(Activity activity) } } } + + // Remove prompt data from InvokeAgent scopes if configured + if (!_options.SendPromptInInvokeAgentScopes) + { + if (activity.OperationName == InvokeAgentScope.OperationName || + (activity.DisplayName != null && activity.DisplayName.StartsWith(InvokeAgentScope.OperationName))) + { + // Remove the gen_ai.input.messages tag to prevent sending prompt content + if (activity.Tags.Any(tag => tag.Key == OpenTelemetryConstants.GenAiInputMessagesKey)) + { + activity.SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, null); + } + } + } } } diff --git a/src/Observability/Extensions/OpenAI/OpenAISpanProcessorOptions.cs b/src/Observability/Extensions/OpenAI/OpenAISpanProcessorOptions.cs new file mode 100644 index 00000000..37caade6 --- /dev/null +++ b/src/Observability/Extensions/OpenAI/OpenAISpanProcessorOptions.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.Observability.Extensions.OpenAI; + +/// +/// Configuration options for OpenAI span processing. +/// +public class OpenAISpanProcessorOptions +{ + /// + /// Gets or sets a value indicating whether to send LLM prompt content in InvokeAgent scopes. + /// When set to false, the gen_ai.input.messages tag will be removed from InvokeAgent spans + /// to prevent sensitive prompt data from being recorded in telemetry. + /// Defaults to true for backward compatibility. + /// + public bool SendPromptInInvokeAgentScopes { get; set; } = true; +} diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/Microsoft.Agents.A365.Observability.Extension.Tests.csproj b/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/Microsoft.Agents.A365.Observability.Extension.Tests.csproj index 5d914e04..ee36da57 100644 --- a/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/Microsoft.Agents.A365.Observability.Extension.Tests.csproj +++ b/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/Microsoft.Agents.A365.Observability.Extension.Tests.csproj @@ -22,5 +22,6 @@ + diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/OpenAISpanProcessorTests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/OpenAISpanProcessorTests.cs new file mode 100644 index 00000000..d5fa82f1 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Extension.Tests/OpenAISpanProcessorTests.cs @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using Microsoft.Agents.A365.Observability.Extensions.OpenAI; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; + +namespace Microsoft.Agents.A365.Observability.Extension.Tests +{ + [TestClass] + public class OpenAISpanProcessorTests + { + private const string InvokeAgentOperationName = "invoke_agent"; + private const string OpenAISourceName = "OpenAI.Test"; + + [TestMethod] + public void OpenAISpanProcessor_SendPromptInInvokeAgentScopes_DefaultsToTrue() + { + // Arrange + var options = new OpenAISpanProcessorOptions(); + + // Assert + Assert.IsTrue(options.SendPromptInInvokeAgentScopes, "SendPromptInInvokeAgentScopes should default to true for backward compatibility"); + } + + [TestMethod] + public void OpenAISpanProcessor_WithDefaultOptions_PreservesPromptInInvokeAgentScope() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = true }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Test prompt content"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNotNull(promptTag.Value, "Prompt should be preserved when SendPromptInInvokeAgentScopes is true"); + Assert.AreEqual("Test prompt content", promptTag.Value); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_RemovesPromptFromInvokeAgentScope() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Sensitive prompt content"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNull(promptTag.Value, "Prompt should be removed when SendPromptInInvokeAgentScopes is false"); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_RemovesPromptFromInvokeAgentScopeWithAgentName() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity("invoke_agent MyAgent") + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Sensitive prompt content"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNull(promptTag.Value, "Prompt should be removed from invoke_agent scope with agent name"); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_PreservesOtherTags() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Sensitive prompt") + .SetTag(OpenTelemetryConstants.GenAiOutputMessagesKey, "Response content") + .SetTag(OpenTelemetryConstants.GenAiAgentIdKey, "agent-123") + .SetTag(OpenTelemetryConstants.GenAiConversationIdKey, "conv-456"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + var outputTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiOutputMessagesKey); + var agentIdTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiAgentIdKey); + var conversationIdTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiConversationIdKey); + + Assert.IsNull(promptTag.Value, "Prompt should be removed"); + Assert.AreEqual("Response content", outputTag.Value, "Output messages should be preserved"); + Assert.AreEqual("agent-123", agentIdTag.Value, "Agent ID should be preserved"); + Assert.AreEqual("conv-456", conversationIdTag.Value, "Conversation ID should be preserved"); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_DoesNotAffectNonInvokeAgentScopes() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity("execute_inference") + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Inference prompt content"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNotNull(promptTag.Value, "Prompt should be preserved for non-InvokeAgent scopes"); + Assert.AreEqual("Inference prompt content", promptTag.Value); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_DoesNotAffectExecuteToolScopes() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity("execute_tool") + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Tool input content"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNotNull(promptTag.Value, "Prompt should be preserved for execute_tool scopes"); + Assert.AreEqual("Tool input content", promptTag.Value); + } + + [TestMethod] + public void OpenAISpanProcessor_WithNullOptions_UsesDefaultBehavior() + { + // Arrange + var processor = new OpenAISpanProcessor(null!); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Test prompt"); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNotNull(promptTag.Value, "Prompt should be preserved with null options (defaults to true)"); + Assert.AreEqual("Test prompt", promptTag.Value); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_HandlesActivityWithoutPromptTag() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiAgentIdKey, "agent-123"); + activity.Start(); + + // Act - should not throw + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNull(promptTag.Value, "Prompt tag should remain null when it was never set"); + } + + [TestMethod] + public void OpenAISpanProcessor_WithSuppressOption_HandlesEmptyPromptTag() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, string.Empty); + activity.Start(); + + // Act + processor.OnEnd(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + // After setting to null, the tag might still exist but with null value + Assert.IsTrue(promptTag.Value == null || string.IsNullOrEmpty(promptTag.Value as string), + "Empty prompt should be removed or set to null"); + } + + [TestMethod] + public void OpenAISpanProcessor_OnStart_DoesNotModifyActivity() + { + // Arrange + var options = new OpenAISpanProcessorOptions { SendPromptInInvokeAgentScopes = false }; + var processor = new OpenAISpanProcessor(options); + + using var activity = new Activity(InvokeAgentOperationName) + .SetTag(OpenTelemetryConstants.GenAiInputMessagesKey, "Test prompt"); + activity.Start(); + + // Act + processor.OnStart(activity); + + // Assert + var promptTag = activity.Tags.FirstOrDefault(t => t.Key == OpenTelemetryConstants.GenAiInputMessagesKey); + Assert.IsNotNull(promptTag.Value, "OnStart should not modify the activity"); + Assert.AreEqual("Test prompt", promptTag.Value); + } + } +}