[Observability] Truncate spans larger than 250 KB

src/Observability/Runtime/Common/ExportFormatter.cs

@@ -2,14 +2,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // ------------------------------------------------------------------------------
 
+using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes;
 using OpenTelemetry.Resources;
 using System;
-using System.Linq;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using Microsoft.Agents.A365.Observability.Runtime.DTOs;
 
 namespace Microsoft.Agents.A365.Observability.Runtime.Common
 {
@@ -18,13 +19,25 @@ namespace Microsoft.Agents.A365.Observability.Runtime.Common
     /// </summary>
     public class ExportFormatter
     {
+        private const int MaxSpanSizeBytes = 250 * 1024;
+        private static readonly string[] LargePayloadAttributeKeys = new[]
+        {
+            OpenTelemetryConstants.GenAiToolArgumentsKey,
+            OpenTelemetryConstants.GenAiEventContent,
+            OpenTelemetryConstants.GenAiInputMessagesKey,
+            OpenTelemetryConstants.GenAiOutputMessagesKey,
+            AutoInstrumentationConstants.GenAiInvocationInputKey,
+            AutoInstrumentationConstants.GenAiInvocationOutputKey
+        };
+
         /// <summary>
         /// Formats a collection of Activity spans into an OTLP JSON payload compatible with the Agent 365 Observability ingestion service.
         /// </summary>
         /// <param name="activities">The collection of Activity spans to be formatted into the OTLP payload.</param>
         /// <param name="resource">The OpenTelemetry resource associated with the spans, containing resource attributes.</param>
+        /// <param name="logInformation">An optional logger for informational messages.</param>
         /// <returns>A JSON string representing the OTLP payload for the provided activities and resource.</returns>
-        public static string FormatMany(IEnumerable<Activity> activities, Resource resource)
+        public static string FormatMany(IEnumerable<Activity> activities, Resource resource, Action<string>? logInformation = null)
         {
             var resourceAttributes = GetResourceAttributes(resource);
             var serviceName = GetServiceName(resource);
@@ -40,7 +53,7 @@ public static string FormatMany(IEnumerable<Activity> activities, Resource resou
                     spans = new List<OtlpSpan>();
                     scopeMap[key] = spans;
                 }
-                spans.Add(BuildOtlpSpan(activity));
+                spans.Add(BuildOtlpSpanWithTruncation(activity: activity, logInformation: logInformation));
             }
 
             var scopeSpans = new List<ScopeSpans>(scopeMap.Count);
@@ -148,6 +161,52 @@ public static string FormatLogData(IDictionary<string, object?> data)
             return resource.Attributes.FirstOrDefault(a => a.Key == "service.version").Value?.ToString();
         }
 
+        private static OtlpSpan BuildOtlpSpanWithTruncation(Activity activity, Action<string>? logInformation = null)
+        {
+            var span = BuildOtlpSpan(activity);
+
+            if (span.Attributes == null)
+                return span;
+
+            // Check initial size
+            if (Encoding.UTF8.GetByteCount(ExportFormatter.SerializePayload(span)) <= ExportFormatter.MaxSpanSizeBytes)
+                return span;
+
+            // Gather key sizes
+            var keySizes = new List<(string Key, int Size, string? Value)>();
+            foreach (var key in ExportFormatter.LargePayloadAttributeKeys)
+            {
+                if (span.Attributes.TryGetValue(key, out var valueObj))
+                {
+                    var value = valueObj as string;
+                    int size = !string.IsNullOrEmpty(value) ? Encoding.UTF8.GetByteCount(value) : 0;
+                    keySizes.Add((key, size, value));
+                    logInformation?.Invoke($"Activity '{activity.DisplayName}': Key '{key}' size = {size / 1024} KB.");
+                }
+            }
+
+            // Sort keys by size descending
+            var sorted = keySizes
+                .Where(k => !string.IsNullOrEmpty(k.Value) && k.Size > 0)
+                .OrderByDescending(k => k.Size)
+                .ToList();
+
+            foreach (var (key, size, _) in sorted)
+            {
+                span.Attributes[key] = "TRUNCATED";
+                logInformation?.Invoke($"Truncated '{key}' in activity '{activity.DisplayName}' to reduce size. Previous size: {size / 1024} KB.");
+
+                // Re-check size after each truncation
+                var json = SerializePayload(span);
+                if (Encoding.UTF8.GetByteCount(json) <= ExportFormatter.MaxSpanSizeBytes)
+                {
+                    break;
+                }
+            }
+
+            return span;
+        }
+
         private static OtlpSpan BuildOtlpSpan(Activity activity)
         {
             return new OtlpSpan

src/Observability/Runtime/Tracing/Exporters/Agent365ExporterCore.cs

@@ -110,7 +110,7 @@ public static async Task<ExportResult> ExportBatchCoreAsync(
             foreach (var g in groups)
             {
                 var (tenantId, agentId, activities) = g;
-                var json = ExportFormatter.FormatMany(activities, resource);
+                var json = ExportFormatter.FormatMany(activities, resource, logInformation);
                 using var content = new StringContent(json, Encoding.UTF8, "application/json");
 
                 var ppapiDiscovery = new PowerPlatformApiDiscovery(options.ClusterCategory);

src/Observability/Runtime/Tracing/Scopes/AutoInstrumentationConstants.cs

@@ -0,0 +1,20 @@
+// ------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ------------------------------------------------------------------------------
+
+namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes
+{
+    /// <summary>
+    /// Constants used for auto-instrumentation.
+    /// </summary>
+    public static class AutoInstrumentationConstants
+    {
+        /// <summary> The key for the input to a GenAI agent invocation. </summary>
+        /// <remarks> Set by the Semantic Kernel OpenTelemetry integration for agent invocations.</remarks>
+        public const string GenAiInvocationInputKey = "gen_ai.agent.invocation_input";
+
+        /// <summary> The key for the output of a GenAI agent invocation. </summary>
+        /// <remarks> Set by the Semantic Kernel OpenTelemetry integration for agent invocations.</remarks>
+        public const string GenAiInvocationOutputKey = "gen_ai.agent.invocation_output";
+    }
+}

src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs

@@ -346,6 +346,102 @@ public async Task AddTracing_NestedScopes_AllExporterRequestsReceived()
             allOperationNames.Should().Contain(new[] { "invoke_agent", "execute_tool", InferenceOperationType.Chat.ToString() }, "All three nested scopes should be exported, even if batched in fewer requests.");
         }
 
+        [TestMethod]
+        public async Task Exporter_Truncates_Scope()
+        {
+            // Arrange
+            this.SetupExporterTest();
+            this._receivedRequest = false;
+            this._receivedContent = null;
+
+            // Create a sample text file >250KB and base64 encode it
+            var tempFile = Path.GetTempFileName();
+            var fileBytes = new byte[300 * 1024]; // 300KB
+            new Random(42).NextBytes(fileBytes);
+            await File.WriteAllBytesAsync(tempFile, fileBytes);
+            var base64 = Convert.ToBase64String(await File.ReadAllBytesAsync(tempFile));
+            File.Delete(tempFile);
+
+            var agentDetails = new AgentDetails(
+                agentId: Guid.NewGuid().ToString(),
+                agentName: "Test Agent",
+                agentDescription: "Agent for truncation test.",
+                agentAUID: Guid.NewGuid().ToString(),
+                agentUPN: "testagent@contoso.com",
+                agentBlueprintId: Guid.NewGuid().ToString(),
+                tenantId: Guid.NewGuid().ToString());
+            var tenantDetails = new TenantDetails(Guid.NewGuid());
+            var endpoint = new Uri("https://test-endpoint");
+            var invokeAgentDetails = new InvokeAgentDetails(details: agentDetails, endpoint: endpoint);
+            var request = new Request(
+                content: "Test request content",
+                executionType: ExecutionType.HumanToAgent,
+                sourceMetadata: new SourceMetadata(name: "test", id: "test-id"));
+
+            var toolCallDetails = new ToolCallDetails(
+                toolName: "LargeFileTool",
+                arguments: base64,
+                toolCallId: "call-123",
+                description: "Test tool with large file content",
+                toolType: "file-upload",
+                endpoint: endpoint);
+
+            // Act: Start nested scopes
+            using (var agentScope = InvokeAgentScope.Start(invokeAgentDetails, tenantDetails, request))
+            {
+                agentScope.RecordInputMessages(new[] { "Agent input" });
+                agentScope.RecordOutputMessages(new[] { "Agent output" });
+                using (var toolScope = ExecuteToolScope.Start(toolCallDetails, agentDetails, tenantDetails))
+                {
+                    toolScope.RecordResponse("Tool response");
+                }
+            }
+
+            // Wait for export
+            var timeout = TimeSpan.FromSeconds(10);
+            var start = DateTime.UtcNow;
+            while (!this._receivedRequest && DateTime.UtcNow - start < timeout)
+            {
+                await Task.Delay(500).ConfigureAwait(false);
+            }
+
+            this._receivedRequest.Should().BeTrue("Exporter should make the expected HTTP request.");
+            this._receivedContent.Should().NotBeNull("Exporter should send a request body.");
+
+            // Assert: Find both activities in the exported payload
+            using var doc = JsonDocument.Parse(this._receivedContent!);
+            var root = doc.RootElement;
+            var spans = root
+                .GetProperty("resourceSpans")[0]
+                .GetProperty("scopeSpans")[0]
+                .GetProperty("spans")
+                .EnumerateArray();
+
+            bool foundInvokeAgent = false;
+            bool foundExecuteTool = false;
+            foreach (var span in spans)
+            {
+                var attrs = span.GetProperty("attributes");
+                var opName = this.GetAttribute(attrs, "gen_ai.operation.name");
+                if (opName == "invoke_agent")
+                {
+                    foundInvokeAgent = true;
+                    // Should NOT be truncated
+                    var input = this.GetAttribute(attrs, "gen_ai.input.messages");
+                    input.Should().Be("Agent input");
+                }
+                if (opName == "execute_tool")
+                {
+                    foundExecuteTool = true;
+                    // Should be truncated
+                    var args = this.GetAttribute(attrs, "gen_ai.tool.arguments");
+                    args.Should().Be("TRUNCATED");
+                }
+            }
+            foundInvokeAgent.Should().BeTrue();
+            foundExecuteTool.Should().BeTrue();
+        }
+
         private class TestHttpMessageHandler : HttpMessageHandler
         {
             private Func<HttpRequestMessage, HttpResponseMessage> _handler;

src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Common/ExportFormatterTests.cs

@@ -483,4 +483,102 @@ public void FormatLogData_WithMissingOptionalFields_ProducesDefaults()
         var attrs = root.GetProperty("Attributes");
         attrs.GetProperty("key").GetString().Should().Be("val");
     }
-}
+
+    #region ExportFormatter FormatMany Truncation Tests
+
+    [TestMethod]
+    public void FormatMany_DoesNothing_WhenUnderLimit()
+    {
+        // Arrange
+        using var activity = CreateActivity("tenant-1", "agent-1");
+        activity.SetTag("gen_ai.tool.arguments", new string('a', 1024)); // 1KB
+        var resource = ResourceBuilder.CreateEmpty().Build();
+        var logs = new List<string>();
+
+        // Act
+        var json = ExportFormatter.FormatMany(new[] { activity }, resource, logs.Add);
+
+        // Assert
+        var doc = JsonDocument.Parse(json);
+        var resourceSpans = doc.RootElement.GetProperty("resourceSpans");
+        var scopeSpans = resourceSpans[0].GetProperty("scopeSpans");
+        var span = scopeSpans[0].GetProperty("spans")[0];
+        span.GetProperty("attributes").GetProperty("gen_ai.tool.arguments").GetString().Should().NotBe("TRUNCATED");
+        logs.Should().NotContain(l => l.Contains("Truncated"));
+    }
+
+    [TestMethod]
+    public void FormatMany_TruncatesSingleLargeKey()
+    {
+        // Arrange
+        using var activity = CreateActivity("tenant-1", "agent-2");
+        activity.SetTag("gen_ai.tool.arguments", new string('b', 300 * 1024)); // 300KB
+        var resource = ResourceBuilder.CreateEmpty().Build();
+        var logs = new List<string>();
+
+        // Act
+        var json = ExportFormatter.FormatMany(new[] { activity }, resource, logs.Add);
+
+        // Assert
+        var doc = JsonDocument.Parse(json);
+        var resourceSpans = doc.RootElement.GetProperty("resourceSpans");
+        var scopeSpans = resourceSpans[0].GetProperty("scopeSpans");
+        var span = scopeSpans[0].GetProperty("spans")[0];
+        span.GetProperty("attributes").GetProperty("gen_ai.tool.arguments").GetString().Should().Be("TRUNCATED");
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.tool.arguments' size = "));
+        logs.Should().Contain(l => l.Contains("Truncated 'gen_ai.tool.arguments'"));
+    }
+
+    [TestMethod]
+    public void FormatMany_TruncatesMultipleKeys_LargestFirst()
+    {
+        // Arrange
+        using var activity = CreateActivity("tenant-1", "agent-1");
+        activity.SetTag("gen_ai.tool.arguments", new string('c', 200 * 1024));
+        activity.SetTag("gen_ai.event.content", new string('d', 100 * 1024));
+        var resource = ResourceBuilder.CreateEmpty().Build();
+        var logs = new List<string>();
+
+        // Act
+        var json = ExportFormatter.FormatMany(new[] { activity }, resource, logs.Add);
+
+        // Assert
+        var doc = JsonDocument.Parse(json);
+        var resourceSpans = doc.RootElement.GetProperty("resourceSpans");
+        var scopeSpans = resourceSpans[0].GetProperty("scopeSpans");
+        var span = scopeSpans[0].GetProperty("spans")[0];
+        var attr = span.GetProperty("attributes");
+        attr.GetProperty("gen_ai.tool.arguments").GetString().Should().Be("TRUNCATED");
+        logs.Should().Contain(l => l.Contains("Truncated 'gen_ai.tool.arguments'"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.tool.arguments' size = "));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.event.content' size = "));
+    }
+
+    [TestMethod]
+    public void FormatMany_LogsAllKeySizes()
+    {
+        // Arrange
+        using var activity = CreateActivity("tenant-1", "agent-1");
+        activity.SetTag("gen_ai.tool.arguments", new string('x', 100 * 1024));
+        activity.SetTag("gen_ai.event.content", new string('y', 125 * 1024));
+        activity.SetTag("gen_ai.input.messages", new string('z', 75 * 1024));
+        activity.SetTag("gen_ai.agent.invocation_input", new string('z', 0));
+        activity.SetTag("gen_ai.agent.invocation_output", new string('z', 0));
+        activity.SetTag("gen_ai.output.messages", new string('z', 0));
+        var resource = ResourceBuilder.CreateEmpty().Build();
+        var logs = new List<string>();
+
+        // Act
+        ExportFormatter.FormatMany(new[] { activity }, resource, logs.Add);
+
+        // Assert
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.tool.arguments' size = 100"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.event.content' size = 125"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.input.messages' size = 75"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.agent.invocation_input' size = 0"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.agent.invocation_output' size = 0"));
+        logs.Should().Contain(l => l.Contains("Key 'gen_ai.output.messages' size = 0"));
+    }
+
+    #endregion
+}

src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Exporters/Agent365ExporterTests.cs

@@ -2,7 +2,6 @@
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters;
 using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes;
-using Microsoft.VisualStudio.TestTools.UnitTesting;
 using OpenTelemetry;
 using OpenTelemetry.Resources;
 using System.Diagnostics;