Address observability security review findings

packages/agents-a365-observability-extensions-langchain/src/tracer.ts

@@ -4,12 +4,13 @@
 import { context, trace, Span, SpanKind, SpanStatusCode, Tracer } from "@opentelemetry/api";
 import { BaseTracer, Run } from "@langchain/core/tracers/base";
 import { isTracingSuppressed } from "@opentelemetry/core";
-import { logger, OpenTelemetryConstants } from "@microsoft/agents-a365-observability";
+import { logger, OpenTelemetryConstants, defaultObservabilityConfigurationProvider } from "@microsoft/agents-a365-observability";
 import * as Utils from "./Utils";
 
 type RunWithSpan = { run: Run; span: Span; startTime: number; lastAccessTime: number };
 
 export class LangChainTracer extends BaseTracer {
+  private static readonly MAX_RUNS = 10_000;
   private tracer: Tracer;
   private runs: Record<string, RunWithSpan> = {};
   private parentByRunId: Record<string, string | undefined> = {};
@@ -59,6 +60,11 @@ export class LangChainTracer extends BaseTracer {
       spanName = `${operation} ${Utils.getModel(run) || run.name}`.trim();
     }
 
+    if (Object.keys(this.runs).length >= LangChainTracer.MAX_RUNS) {
+      logger.warn(`[LangChainTracer] Max runs (${LangChainTracer.MAX_RUNS}) reached, skipping span`);
+      return;
+    }
+
     const startTime = run.start_time ?? Date.now();
     const span = this.tracer.startSpan(spanName, {
       kind: SpanKind.INTERNAL,
@@ -77,11 +83,13 @@ export class LangChainTracer extends BaseTracer {
     const operation = Utils.getOperationType(run);
     if (run.tags?.includes("langsmith:hidden") || run.name?.startsWith("Branch") || operation === "unknown") {
       logger.info(`Skipping internal run: ${run.name} (parent: ${run.parent_run_id})`);
+      delete this.parentByRunId[run.id];
       return;
     }
 
     const entry = this.runs[run.id];
     if (!entry) {
+      delete this.parentByRunId[run.id];
       return;
     }
 
@@ -91,7 +99,8 @@ export class LangChainTracer extends BaseTracer {
 
       if (run.error) {
         span.setStatus({ code: SpanStatusCode.ERROR });
-        span.setAttribute(OpenTelemetryConstants.ERROR_MESSAGE_KEY, String(run.error));
+        const errorMsg = String(run.error);
+        span.setAttribute(OpenTelemetryConstants.ERROR_MESSAGE_KEY, errorMsg.length > 1024 ? errorMsg.substring(0, 1024) + '...[truncated]' : errorMsg);
 
       } else {
         span.setStatus({ code: SpanStatusCode.OK });
@@ -100,15 +109,20 @@ export class LangChainTracer extends BaseTracer {
       // Set all attributes
       Utils.setOperationTypeAttribute(operation, span);
       Utils.setAgentAttributes(run, span);
-      Utils.setToolAttributes(run, span);
-      Utils.setInputMessagesAttribute(run, span);
-      Utils.setOutputMessagesAttribute(run, span);
-      Utils.setSystemInstructionsAttribute(run, span);
       Utils.setModelAttribute(run, span);
       Utils.setProviderNameAttribute(run, span);
       Utils.setSessionIdAttribute(run, span);
       Utils.setTokenAttributes(run, span);
 
+      // Content attributes gated by content recording setting
+      const contentRecording = defaultObservabilityConfigurationProvider.getConfiguration().isContentRecordingEnabled;
+      if (contentRecording) {
+        Utils.setToolAttributes(run, span);
+        Utils.setInputMessagesAttribute(run, span);
+        Utils.setOutputMessagesAttribute(run, span);
+        Utils.setSystemInstructionsAttribute(run, span);
+      }
+
     } catch (error) {
       logger.error(`[LangChainTracer] Error setting span attributes for run ${run.name}: ${error instanceof Error ? error.message : String(error)}`);
       span.setStatus({ code: SpanStatusCode.ERROR });

packages/agents-a365-observability-extensions-openai/package.json

@@ -49,8 +49,7 @@
     "@microsoft/agents-a365-runtime": "workspace:*",
     "@openai/agents": "catalog:",
     "@opentelemetry/api": "catalog:",
-    "@opentelemetry/instrumentation": "catalog:",
-    "hono": "catalog:"
+    "@opentelemetry/instrumentation": "catalog:"
   },
   "devDependencies": {
     "@eslint/js": "catalog:",
@@ -66,7 +65,13 @@
     "typescript-eslint": "catalog:"
   },
   "peerDependencies": {
-    "@openai/agents": "catalog:"
+    "@openai/agents": "catalog:",
+    "hono": "catalog:"
+  },
+  "peerDependenciesMeta": {
+    "hono": {
+      "optional": true
+    }
   },
   "engines": {
     "node": ">=18.0.0"

packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts

@@ -8,7 +8,7 @@
  */
 
 import { context, trace as OtelTrace, Span as OtelSpan, Tracer as OtelTracer } from '@opentelemetry/api';
-import { OpenTelemetryConstants, InferenceOperationType } from '@microsoft/agents-a365-observability';
+import { OpenTelemetryConstants, InferenceOperationType, defaultObservabilityConfigurationProvider, logger } from '@microsoft/agents-a365-observability';
 import * as Constants from './Constants';
 import * as Utils from './Utils';
 import {
@@ -29,6 +29,7 @@ type ContextToken = unknown;
  */
 export class OpenAIAgentsTraceProcessor implements TracingProcessor {
   private static readonly MAX_HANDOFFS_IN_FLIGHT = 1000;
+  private static readonly MAX_SPANS_IN_FLIGHT = 10_000;
 
   private readonly tracer: OtelTracer;
   private readonly suppressInvokeAgentInput: boolean;
@@ -53,6 +54,19 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
     this.suppressInvokeAgentInput = options?.suppressInvokeAgentInput ?? false;
   }
 
+  private static readonly CONTENT_KEYS = new Set([
+    OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY,
+    OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY,
+    OpenTelemetryConstants.GEN_AI_EVENT_CONTENT,
+    OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY,
+    Constants.GEN_AI_REQUEST_CONTENT_KEY,
+    Constants.GEN_AI_RESPONSE_CONTENT_KEY,
+  ]);
+
+  private static isContentKey(key: string): boolean {
+    return OpenAIAgentsTraceProcessor.CONTENT_KEYS.has(key);
+  }
+
   private getNewKey(spanType: string, key: string): string | null {
     return this.keyMappings.get(`${spanType}${key}`) ?? null;
   }
@@ -96,6 +110,11 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
       return;
     }
 
+    if (this.otelSpans.size >= OpenAIAgentsTraceProcessor.MAX_SPANS_IN_FLIGHT) {
+      logger.warn(`[OpenAIAgentsTraceProcessor] Max spans in flight (${OpenAIAgentsTraceProcessor.MAX_SPANS_IN_FLIGHT}) reached, skipping span`);
+      return;
+    }
+
     const startTime = new Date(startedAt).getTime();
 
     // Find parent span
@@ -219,6 +238,7 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
    * Process response span data
    */
   private processResponseSpanData(otelSpan: OtelSpan, data: SpanData): void {
+    const contentRecording = defaultObservabilityConfigurationProvider.getConfiguration().isContentRecordingEnabled;
     const responseData = data as Record<string, unknown>;
     // Handle both formats: _response/_input (actual format) and response/input (legacy format)
     const responseObj = responseData._response || responseData.response;
@@ -227,7 +247,7 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
       const resp = responseObj as Record<string, unknown>;
 
       // Store the output field for GEN_AI_RESPONSE_CONTENT_KEY
-      if (resp.output) {
+      if (resp.output && contentRecording) {
         if (typeof resp.output === 'string') {
           otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, resp.output);
         } else {
@@ -251,7 +271,7 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
       otelSpan.updateName(`${InferenceOperationType.CHAT} ${modelName}`);
     }
 
-    if (inputObj && !this.suppressInvokeAgentInput) {
+    if (inputObj && !this.suppressInvokeAgentInput && contentRecording) {
       if (typeof inputObj === 'string') {
         try {
           const parsed = JSON.parse(inputObj as string);
@@ -306,14 +326,18 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
    * Process generation span data
    */
   private processGenerationSpanData(otelSpan: OtelSpan, data: SpanData, traceId: string): void {
+    const contentRecording = defaultObservabilityConfigurationProvider.getConfiguration().isContentRecordingEnabled;
     const attrs = Utils.getAttributesFromGenerationSpanData(data);
     Object.entries(attrs).forEach(([key, value]) => {
       const shouldExcludeKey = key === OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY
         || key === Constants.GEN_AI_EXECUTION_PAYLOAD_KEY;
       if (value !== null && value !== undefined && !shouldExcludeKey) {
         const newKey = this.getNewKey(data.type, key);
-        if (newKey !== OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY || !this.suppressInvokeAgentInput) {
-          otelSpan.setAttribute(newKey || key, value as string | number | boolean);
+        const resolvedKey = newKey || key;
+        if (resolvedKey !== OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY || !this.suppressInvokeAgentInput) {
+          if (!OpenAIAgentsTraceProcessor.isContentKey(resolvedKey) || contentRecording) {
+            otelSpan.setAttribute(resolvedKey, value as string | number | boolean);
+          }
         }
       }
     });
@@ -332,12 +356,16 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
    * Process function/tool span data
    */
   private processFunctionSpanData(otelSpan: OtelSpan, data: SpanData, traceId: string): void {
+    const contentRecording = defaultObservabilityConfigurationProvider.getConfiguration().isContentRecordingEnabled;
     const functionData = data as Record<string, unknown>;
     const attrs = Utils.getAttributesFromFunctionSpanData(data);
     Object.entries(attrs).forEach(([key, value]) => {
       if (value !== null && value !== undefined && key !== OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY) {
         const newKey = this.getNewKey(data.type, key);
-        otelSpan.setAttribute(newKey || key, value as string | number | boolean);
+        const resolvedKey = newKey || key;
+        if (!OpenAIAgentsTraceProcessor.isContentKey(resolvedKey) || contentRecording) {
+          otelSpan.setAttribute(resolvedKey, value as string | number | boolean);
+        }
       }
       otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_TOOL_TYPE_KEY, 'function');
     });

packages/agents-a365-observability-hosting/src/caching/AgenticTokenCache.ts

@@ -29,6 +29,8 @@ export class AgenticTokenCache {
     private readonly _map = new Map<string, CacheEntry>();
     private readonly _defaultRefreshSkewMs = 60_000;
     private readonly _defaultMaxTokenAgeMs = 3_600_000;
+    private readonly _maxCacheSize = 10_000;
+    private readonly _maxExpSeconds = 86_400; // 24 hours
     private readonly _keyLocks = new Map<string, Promise<unknown>>();
     private readonly _configProvider: IConfigurationProvider<ObservabilityConfiguration>;
 
@@ -85,6 +87,12 @@ export class AgenticTokenCache {
                     return;
                 }
                 entry = { scopes: effectiveScopes };
+                if (this._map.size >= this._maxCacheSize) {
+                    const oldest = this._map.keys().next().value;
+                    if (oldest !== undefined) {
+                        this._map.delete(oldest);
+                    }
+                }
                 this._map.set(key, entry);
             }
             if (!Array.isArray(entry.scopes) || entry.scopes.length === 0) {
@@ -157,7 +165,9 @@ export class AgenticTokenCache {
             const payloadSegment = parts[1];
             const padded = payloadSegment + '='.repeat((4 - (payloadSegment.length % 4)) % 4);
             const json = JSON.parse(Buffer.from(padded, 'base64').toString('utf8')) as { exp?: unknown };
-            return typeof json.exp === 'number' ? json.exp : undefined;
+            if (typeof json.exp !== 'number') return undefined;
+            const maxExp = Math.floor(Date.now() / 1000) + this._maxExpSeconds;
+            return Math.min(json.exp, maxExp);
         } catch {
             return undefined;
         }

packages/agents-a365-observability-hosting/src/utils/ScopeUtils.ts

@@ -15,6 +15,7 @@ import {
   InferenceDetails,
   InvokeAgentDetails,
   ToolCallDetails,
+  defaultObservabilityConfigurationProvider,
 } from '@microsoft/agents-a365-observability';
 import { resolveEmbodiedAgentIds } from './TurnContextUtils';
 
@@ -26,7 +27,7 @@ export class ScopeUtils {
 
 
   private static setInputMessageTags(scope: InvokeAgentScope | InferenceScope, turnContext: TurnContext): InvokeAgentScope | InferenceScope {
-    if (turnContext?.activity?.text) {
+    if (turnContext?.activity?.text && defaultObservabilityConfigurationProvider.getConfiguration().isContentRecordingEnabled) {
       scope.recordInputMessages([turnContext.activity.text]);
     }
     return scope;

packages/agents-a365-observability/src/configuration/ObservabilityConfiguration.ts

@@ -61,4 +61,15 @@ export class ObservabilityConfiguration extends RuntimeConfiguration {
       ?? process.env.A365_OBSERVABILITY_LOG_LEVEL
       ?? 'none';
   }
+
+  /**
+   * Whether content recording is enabled for telemetry spans.
+   * When disabled, sensitive content (prompts, completions, tool I/O, system instructions)
+   * is not recorded as span attributes.
+   */
+  get isContentRecordingEnabled(): boolean {
+    const result = this.observabilityOverrides.isContentRecordingEnabled?.();
+    if (result !== undefined) return result;
+    return RuntimeConfiguration.parseEnvBoolean(process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED);
+  }
 }

packages/agents-a365-observability/src/configuration/ObservabilityConfigurationOptions.ts

@@ -61,4 +61,15 @@ export type ObservabilityConfigurationOptions = RuntimeConfigurationOptions & {
    * @default 'none'
    */
   observabilityLogLevel?: () => string;
+
+  /**
+   * Override to enable/disable content recording in telemetry spans.
+   * When disabled, sensitive content (prompts, completions, tool I/O, system instructions)
+   * is not recorded as span attributes.
+   *
+   * @returns `true` to enable content recording, `false` to disable.
+   * @envvar AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED - 'true' to enable.
+   * @default false
+   */
+  isContentRecordingEnabled?: () => boolean;
 };

packages/agents-a365-observability/src/tracing/exporter/Agent365Exporter.ts

@@ -171,8 +171,8 @@ export class Agent365Exporter implements SpanExporter {
     // Select endpoint path based on S2S flag (includes tenantId in path)
     const endpointRelativePath =
       this.options.useS2SEndpoint
-        ? `/observabilityService/tenants/${tenantId}/agents/${agentId}/traces`
-        : `/observability/tenants/${tenantId}/agents/${agentId}/traces`;
+        ? `/observabilityService/tenants/${encodeURIComponent(tenantId)}/agents/${encodeURIComponent(agentId)}/traces`
+        : `/observability/tenants/${encodeURIComponent(tenantId)}/agents/${encodeURIComponent(agentId)}/traces`;
 
     let url: string;
     const domainOverride = getAgent365ObservabilityDomainOverride(this.configProvider);
@@ -260,7 +260,7 @@ export class Agent365Exporter implements SpanExporter {
         // Retry transient errors
         if ([408, 429].includes(response.status) || (response.status >= 500 && response.status < 600)) {
           if (attempt < DEFAULT_MAX_RETRIES) {
-            const sleepMs = 200 * (attempt + 1);
+            const sleepMs = 200 * (attempt + 1) + Math.floor(Math.random() * 100);
             logger.warn(`[Agent365Exporter] Transient error ${response.status}, correlation ID: ${correlationId}, retrying after ${sleepMs}ms`);
             await this.sleep(sleepMs);
             continue;

tests/observability/extension/hosting/scope-utils.test.ts

@@ -65,10 +65,12 @@ function makeTurnContext(
 describe('ScopeUtils.populateFromTurnContext', () => {
   let spy: jest.SpyInstance;
   beforeEach(() => {
+    process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED = 'true';
     spy = jest.spyOn(OpenTelemetryScope.prototype as any, 'setTagMaybe');
    });
 
   afterEach(() => {
+    delete process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED;
     spy.mockRestore();
   });
 

tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts

@@ -466,6 +466,7 @@ describe('OpenAIAgentsTraceProcessor', () => {
     };
 
     beforeEach(() => {
+      process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED = 'true';
       spansByName = {};
       tracerSpy = jest.spyOn(tracer as any, 'startSpan').mockImplementation((...args: unknown[]) => {
         const name = args[0] as string;
@@ -476,6 +477,7 @@ describe('OpenAIAgentsTraceProcessor', () => {
     });
 
     afterEach(() => {
+      delete process.env.AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED;
       tracerSpy.mockRestore();
     });