From a02dd86d6aa97a6f863cdcb224a9bb5d1ee8028d Mon Sep 17 00:00:00 2001 From: jsl517 Date: Fri, 19 Dec 2025 12:49:08 -0800 Subject: [PATCH 1/5] Add option to auto instrumentation to prevent sending the LLM prompt on InvokeAgent scopes. --- .../src/OpenAIAgentsTraceInstrumentor.ts | 10 +- .../src/OpenAIAgentsTraceProcessor.ts | 31 +++--- .../openai/OpenAIAgentsTraceProcessor.test.ts | 104 ++++++++++++++++++ 3 files changed, 131 insertions(+), 14 deletions(-) diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts index 5d81256d..db3e217a 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts @@ -21,6 +21,12 @@ export interface OpenAIAgentsInstrumentationConfig extends InstrumentationConfig enabled?: boolean; tracerName?: string; tracerVersion?: string; + /** + * When false, auto-instrumentation will not attach LLM prompt content + * Defaults to true. + */ + sendPromptInInvokeAgentScopes?: boolean; + } /** @@ -93,7 +99,9 @@ export class OpenAIAgentsTraceInstrumentor extends InstrumentationBase = new Map(); private readonly otelSpans: Map = new Map(); private readonly tokens: Map = new Map(); @@ -47,8 +48,9 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { ['generation' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY], ]); - constructor(tracer: OtelTracer) { + constructor(tracer: OtelTracer, options?: { sendPromptInInvokeAgentScopes?: boolean }) { this.tracer = tracer; + this.sendPromptInInvokeAgentScopes = options?.sendPromptInInvokeAgentScopes ?? true; } private getNewKey(spanType: string, key: string): string | null { @@ -244,18 +246,18 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { const modelName = attrs[OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY] ?? ''; otelSpan.updateName(`${InferenceOperationType.CHAT} ${modelName}`); - } if (inputObj) { - if (typeof inputObj === 'string') { - otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, inputObj); - } else if (Array.isArray(inputObj)) { - // Store the complete _input structure as JSON - otelSpan.setAttribute( - OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, - JSON.stringify(inputObj) - ); + if (this.sendPromptInInvokeAgentScopes) { + if (typeof inputObj === 'string') { + otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, inputObj); + } else if (Array.isArray(inputObj)) { + // Store the complete _input structure as JSON + otelSpan.setAttribute( + OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, + JSON.stringify(inputObj) + ); // Get attributes but filter out unwanted ones const attrs = Utils.getAttributesFromInput(inputObj); @@ -263,8 +265,9 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { if (value !== null && value !== undefined && key !== Constants.GEN_AI_REQUEST_CONTENT_KEY) { otelSpan.setAttribute(key, value as string | number | boolean); - } - }); + } + }); + } } } } @@ -279,7 +282,9 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { || key === Constants.GEN_AI_EXECUTION_PAYLOAD_KEY; if (value !== null && value !== undefined && !shouldExcludeKey) { const newKey = this.getNewKey(data.type, key); - otelSpan.setAttribute(newKey || key, value as string | number | boolean); + if (newKey !== OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY || this.sendPromptInInvokeAgentScopes) { + otelSpan.setAttribute(newKey || key, value as string | number | boolean); + } } }); diff --git a/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts b/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts index 7a45835b..dde15d62 100644 --- a/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts +++ b/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts @@ -9,6 +9,7 @@ import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; import { Tracer } from '@opentelemetry/api'; +import { OpenTelemetryConstants } from '@microsoft/agents-a365-observability'; import { OpenAIAgentsTraceProcessor } from '@microsoft/agents-a365-observability-extensions-openai'; import { ObservabilityManager } from '@microsoft/agents-a365-observability'; import { trace } from '@opentelemetry/api'; @@ -447,4 +448,107 @@ describe('OpenAIAgentsTraceProcessor', () => { }); }); }); + + describe('Prompt Suppression in InvokeAgent traces', () => { + let spansByName: Record; + let tracerSpy: jest.SpyInstance; + + const createMockSpan = (name: string) => { + const attrs: Array<[string, unknown]> = []; + return { + setAttribute: jest.fn((k: string, v: unknown) => { attrs.push([k, v]); }), + updateName: jest.fn(), + setStatus: jest.fn(), + end: jest.fn(), + spanContext: jest.fn(() => ({ traceId: 'tid-' + name, spanId: 'sid-' + name })), + _attrs: attrs, + }; + }; + + beforeEach(() => { + spansByName = {}; + tracerSpy = jest.spyOn(tracer as any, 'startSpan').mockImplementation((...args: unknown[]) => { + const name = args[0] as string; + const s = createMockSpan(name); + spansByName[name] = s; + return s; + }); + }); + + afterEach(() => { + tracerSpy.mockRestore(); + }); + + it('does not record GEN_AI_INPUT_MESSAGES when disabled', async () => { + const processor = new OpenAIAgentsTraceProcessor(tracer, { sendPromptInInvokeAgentScopes: false }); + const traceData = { traceId: 'trace-suppress', name: 'Agent' } as any; + await processor.onTraceStart(traceData); + + const agentSpan = { + spanId: 'agent-span', traceId: 'trace-suppress', startedAt: new Date().toISOString(), + spanData: { type: 'agent' as const, name: 'agent-node' }, + } as any; + await processor.onSpanStart(agentSpan); + await processor.onSpanEnd(agentSpan); + + const genSpan = { + spanId: 'gen-span', traceId: 'trace-suppress', startedAt: new Date().toISOString(), + spanData: { type: 'generation' as const, name: 'Generate', model: 'gpt-4', input: 'Hello prompt' }, + } as any; + await processor.onSpanStart(genSpan); + await processor.onSpanEnd(genSpan); + + const genMock = spansByName['Generate']; + const keys = (genMock._attrs as Array<[string, unknown]>).map(([k]) => k); + expect(keys).not.toContain(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY); + }); + + it('records GEN_AI_INPUT_MESSAGES when enabled (default)', async () => { + const processor = new OpenAIAgentsTraceProcessor(tracer); + const traceData = { traceId: 'trace-allow', name: 'Agent' } as any; + await processor.onTraceStart(traceData); + + const agentSpan = { + spanId: 'agent-span-2', traceId: 'trace-allow', startedAt: new Date().toISOString(), + spanData: { type: 'agent' as const, name: 'agent-node-2' }, + } as any; + await processor.onSpanStart(agentSpan); + await processor.onSpanEnd(agentSpan); + + const genSpan = { + spanId: 'gen-span-2', traceId: 'trace-allow', startedAt: new Date().toISOString(), + spanData: { type: 'generation' as const, name: 'Generate2', model: 'gpt-4', input: 'Hello prompt' }, + } as any; + await processor.onSpanStart(genSpan); + await processor.onSpanEnd(genSpan); + + const genMock = spansByName['Generate2']; + const keys = (genMock._attrs as Array<[string, unknown]>).map(([k]) => k); + expect(keys).toContain(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY); + }); + + it('suppresses input on response spans when disabled', async () => { + const processor = new OpenAIAgentsTraceProcessor(tracer, { sendPromptInInvokeAgentScopes: false }); + const traceData = { traceId: 'trace-resp', name: 'Agent' } as any; + await processor.onTraceStart(traceData); + + const agentSpan = { + spanId: 'agent-span-3', traceId: 'trace-resp', startedAt: new Date().toISOString(), + spanData: { type: 'agent' as const, name: 'agent-node-3' }, + } as any; + await processor.onSpanStart(agentSpan); + await processor.onSpanEnd(agentSpan); + + const respSpan = { + spanId: 'resp-span', traceId: 'trace-resp', startedAt: new Date().toISOString(), + spanData: { type: 'response' as const, name: 'Response', _input: 'Prompt text', _response: { model: 'gpt-4', output: 'ok' } }, + } as any; + await processor.onSpanStart(respSpan); + await processor.onSpanEnd(respSpan); + + const respMock = spansByName['Response']; + const keys = (respMock._attrs as Array<[string, unknown]>).map(([k]) => k); + expect(keys).not.toContain(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY); + }); + }); }); From 472d8c7af5efdf7c4f1a3bf96cdc356c723f57b0 Mon Sep 17 00:00:00 2001 From: jsl517 Date: Fri, 19 Dec 2025 13:28:27 -0800 Subject: [PATCH 2/5] comment --- .../src/OpenAIAgentsTraceInstrumentor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts index db3e217a..2b0b6064 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts @@ -22,11 +22,11 @@ export interface OpenAIAgentsInstrumentationConfig extends InstrumentationConfig tracerName?: string; tracerVersion?: string; /** - * When false, auto-instrumentation will not attach LLM prompt content + * When false, the gen_ai.prompt attribute containing LLM input messages + * will not be attached to spans in InvokeAgent scopes. * Defaults to true. */ sendPromptInInvokeAgentScopes?: boolean; - } /** From 7cdd5e5ef140bd582d18c13c99dd966de4b58c8c Mon Sep 17 00:00:00 2001 From: jsl517 Date: Fri, 19 Dec 2025 16:33:10 -0800 Subject: [PATCH 3/5] comments --- .../src/OpenAIAgentsTraceInstrumentor.ts | 10 +++--- .../src/OpenAIAgentsTraceProcessor.ts | 32 +++++++++---------- .../openai/OpenAIAgentsTraceProcessor.test.ts | 4 +-- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts index 2b0b6064..7e6d40ea 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts @@ -22,11 +22,11 @@ export interface OpenAIAgentsInstrumentationConfig extends InstrumentationConfig tracerName?: string; tracerVersion?: string; /** - * When false, the gen_ai.prompt attribute containing LLM input messages - * will not be attached to spans in InvokeAgent scopes. - * Defaults to true. + * When true, the gen_ai.prompt attribute containing LLM input messages + * will be suppressed and not attached to spans in InvokeAgent scopes. + * Defaults to false. */ - sendPromptInInvokeAgentScopes?: boolean; + suppressInvokeAgentInput?: boolean; } /** @@ -100,7 +100,7 @@ export class OpenAIAgentsTraceInstrumentor extends InstrumentationBase = new Map(); private readonly otelSpans: Map = new Map(); private readonly tokens: Map = new Map(); @@ -48,9 +48,9 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { ['generation' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY], ]); - constructor(tracer: OtelTracer, options?: { sendPromptInInvokeAgentScopes?: boolean }) { + constructor(tracer: OtelTracer, options?: { suppressInvokeAgentInput?: boolean }) { this.tracer = tracer; - this.sendPromptInInvokeAgentScopes = options?.sendPromptInInvokeAgentScopes ?? true; + this.suppressInvokeAgentInput = options?.suppressInvokeAgentInput ?? false; } private getNewKey(spanType: string, key: string): string | null { @@ -248,16 +248,15 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { otelSpan.updateName(`${InferenceOperationType.CHAT} ${modelName}`); } - if (inputObj) { - if (this.sendPromptInInvokeAgentScopes) { - if (typeof inputObj === 'string') { - otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, inputObj); - } else if (Array.isArray(inputObj)) { - // Store the complete _input structure as JSON - otelSpan.setAttribute( - OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, - JSON.stringify(inputObj) - ); + if (inputObj && !this.suppressInvokeAgentInput) { + if (typeof inputObj === 'string') { + otelSpan.setAttribute(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, inputObj); + } else if (Array.isArray(inputObj)) { + // Store the complete _input structure as JSON + otelSpan.setAttribute( + OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, + JSON.stringify(inputObj) + ); // Get attributes but filter out unwanted ones const attrs = Utils.getAttributesFromInput(inputObj); @@ -265,9 +264,8 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { if (value !== null && value !== undefined && key !== Constants.GEN_AI_REQUEST_CONTENT_KEY) { otelSpan.setAttribute(key, value as string | number | boolean); - } - }); - } + } + }); } } } @@ -282,7 +280,7 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { || key === Constants.GEN_AI_EXECUTION_PAYLOAD_KEY; if (value !== null && value !== undefined && !shouldExcludeKey) { const newKey = this.getNewKey(data.type, key); - if (newKey !== OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY || this.sendPromptInInvokeAgentScopes) { + if (newKey !== OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY || !this.suppressInvokeAgentInput) { otelSpan.setAttribute(newKey || key, value as string | number | boolean); } } diff --git a/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts b/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts index dde15d62..ed477be6 100644 --- a/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts +++ b/tests/observability/extension/openai/OpenAIAgentsTraceProcessor.test.ts @@ -480,7 +480,7 @@ describe('OpenAIAgentsTraceProcessor', () => { }); it('does not record GEN_AI_INPUT_MESSAGES when disabled', async () => { - const processor = new OpenAIAgentsTraceProcessor(tracer, { sendPromptInInvokeAgentScopes: false }); + const processor = new OpenAIAgentsTraceProcessor(tracer, { suppressInvokeAgentInput: true }); const traceData = { traceId: 'trace-suppress', name: 'Agent' } as any; await processor.onTraceStart(traceData); @@ -528,7 +528,7 @@ describe('OpenAIAgentsTraceProcessor', () => { }); it('suppresses input on response spans when disabled', async () => { - const processor = new OpenAIAgentsTraceProcessor(tracer, { sendPromptInInvokeAgentScopes: false }); + const processor = new OpenAIAgentsTraceProcessor(tracer, { suppressInvokeAgentInput: true }); const traceData = { traceId: 'trace-resp', name: 'Agent' } as any; await processor.onTraceStart(traceData); From f0733f5bd10cbf63e0830eee8652efb0fefc4de2 Mon Sep 17 00:00:00 2001 From: PengF <126631706+fpfp100@users.noreply.github.com> Date: Fri, 19 Dec 2025 16:41:26 -0800 Subject: [PATCH 4/5] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/OpenAIAgentsTraceProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts index 6efd6a0f..0105689f 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts @@ -50,7 +50,7 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor { constructor(tracer: OtelTracer, options?: { suppressInvokeAgentInput?: boolean }) { this.tracer = tracer; - this.suppressInvokeAgentInput = options?.suppressInvokeAgentInput ?? false; + this.suppressInvokeAgentInput = options?.suppressInvokeAgentInput ?? false; } private getNewKey(spanType: string, key: string): string | null { From a93286e20cca2c9db17c21bb20570b1c1c05573f Mon Sep 17 00:00:00 2001 From: jsl517 Date: Fri, 19 Dec 2025 17:02:50 -0800 Subject: [PATCH 5/5] comments --- .../src/OpenAIAgentsTraceInstrumentor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts index 7e6d40ea..27a7693b 100644 --- a/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts +++ b/packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceInstrumentor.ts @@ -22,7 +22,7 @@ export interface OpenAIAgentsInstrumentationConfig extends InstrumentationConfig tracerName?: string; tracerVersion?: string; /** - * When true, the gen_ai.prompt attribute containing LLM input messages + * When true, the gen_ai.input.messages attribute containing LLM input messages * will be suppressed and not attached to spans in InvokeAgent scopes. * Defaults to false. */