From b40ec74c868ed9fba45149f9973ae022b6f56cb3 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 9 Feb 2026 10:26:55 +0100 Subject: [PATCH 1/3] feat: test `gen_ai.operation.name` to be of a certain pattern for the various span types, defined by a regex --- src/test-cases/checks.ts | 100 +++++++++++++++++++++++++++++---------- src/test-cases/utils.ts | 21 +++++++- 2 files changed, 94 insertions(+), 27 deletions(-) diff --git a/src/test-cases/checks.ts b/src/test-cases/checks.ts index 08d487d..888c2ac 100644 --- a/src/test-cases/checks.ts +++ b/src/test-cases/checks.ts @@ -41,6 +41,63 @@ export interface Check { fn: CheckFunction; } +// ============================================================================= +// Operation Name Patterns +// ============================================================================= +// These patterns are derived from the Sentry backend logic that determines +// gen_ai.operation.type from gen_ai.operation.name. +// +// Reference (Rust code that determines operation type): +// - "agent" type: invoke_agent, create_agent, ai.run.*, ai.pipeline.*, ai.streamText, ai.generateText, ai.generateObject +// - "ai_client" type: *.doStream, *.doGenerate (the actual LLM API calls) +// - "tool" type: execute_tool, ai.toolCall.* +// - "handoff" type: handoff + +/** + * Pattern for agent operation names (gen_ai.operation.name) + * + * Matches: + * - gen_ai.invoke_agent, invoke_agent + * - gen_ai.create_agent, create_agent + * - ai.run.generateText, ai.run.generateObject + * - ai.pipeline.generate_text, ai.pipeline.generate_object, ai.pipeline.stream_text, ai.pipeline.stream_object + * - ai.streamText (but NOT ai.streamText.doStream) + * - ai.generateText (but NOT ai.generateText.doGenerate) + * - ai.generateObject (but NOT ai.generateObject.doGenerate) + */ +export const AGENT_OPERATION_NAME_PATTERN = + /^(gen_ai\.)?(invoke_agent|create_agent)$|^ai\.run\.(generateText|generateObject)$|^ai\.pipeline\.(generate_text|generate_object|stream_text|stream_object)$|^ai\.(streamText|generateText|generateObject)(?!\.do)/; + +/** + * Pattern for ai_client (chat/completion) operation names (gen_ai.operation.name) + * + * Matches: + * - ai.streamText.doStream.* + * - ai.generateText.doGenerate.* + * - ai.generateObject.doGenerate.* + * - chat, completion, generate (legacy) + */ +export const AI_CLIENT_OPERATION_NAME_PATTERN = + /^ai\.(streamText\.doStream|generateText\.doGenerate|generateObject\.doGenerate)|^(gen_ai\.)?(chat|completion|generate)/; + +/** + * Pattern for tool operation names (gen_ai.operation.name) + * + * Matches: + * - gen_ai.execute_tool, execute_tool + * - ai.toolCall.* + */ +export const TOOL_OPERATION_NAME_PATTERN = + /^(gen_ai\.)?(execute_tool|tool|tool_call)$|^ai\.toolCall/; + +/** + * Pattern for handoff operation names (gen_ai.operation.name) + * + * Matches: + * - gen_ai.handoff, handoff + */ +export const HANDOFF_OPERATION_NAME_PATTERN = /^(gen_ai\.)?handoff$/; + // ============================================================================= // Structure Checks // ============================================================================= @@ -122,7 +179,7 @@ export function checkAISpanCount( * Check attributes on chat/completion spans (LLM API calls) * * Validates: - * - gen_ai.operation.name exists + * - gen_ai.operation.name matches AI_CLIENT_OPERATION_NAME_PATTERN * - gen_ai.request.model matches expected model * - gen_ai.request.messages exists * - gen_ai.response.model matches expected pattern @@ -147,7 +204,7 @@ export const checkChatSpanAttributes: Check = { config.modelOverrides?.response || `${requestModel.replace("*", "")}*`; assertAttributes(chatSpans, { - "gen_ai.operation.name": true, + "gen_ai.operation.name": AI_CLIENT_OPERATION_NAME_PATTERN, "gen_ai.request.model": requestModel, "gen_ai.request.messages": true, "gen_ai.response.model": responseModel, @@ -162,6 +219,7 @@ export const checkChatSpanAttributes: Check = { * Check attributes on invoke_agent spans (agent invocations) * * Validates: + * - gen_ai.operation.name matches AGENT_OPERATION_NAME_PATTERN * - gen_ai.agent.name exists * * Fails if no agent spans are found. @@ -175,13 +233,10 @@ export const checkAgentSpanAttributes: Check = { "Should have at least one agent span", ).to.be.greaterThan(0); - // TODO: Add attribute validation once we know what attributes agent spans should have - for (const span of agentSpans) { - expect( - span.data?.["gen_ai.agent.name"], - `Agent span should have gen_ai.agent.name attribute`, - ).to.exist; - } + assertAttributes(agentSpans, { + "gen_ai.operation.name": AGENT_OPERATION_NAME_PATTERN, + "gen_ai.agent.name": true, + }); }, }; @@ -189,6 +244,7 @@ export const checkAgentSpanAttributes: Check = { * Check attributes on tool execution spans * * Validates: + * - gen_ai.operation.name matches TOOL_OPERATION_NAME_PATTERN * - gen_ai.tool.type exists * - gen_ai.tool.name exists * - gen_ai.tool.description exists @@ -204,20 +260,12 @@ export const checkToolSpanAttributes: Check = { "Should have at least one tool span", ).to.be.greaterThan(0); - for (const span of toolSpans) { - expect( - span.data?.["gen_ai.tool.type"], - `Tool span should have gen_ai.tool.type attribute`, - ).to.exist; - expect( - span.data?.["gen_ai.tool.name"], - `Tool span should have gen_ai.tool.name attribute`, - ).to.exist; - expect( - span.data?.["gen_ai.tool.description"], - `Tool span should have gen_ai.tool.description attribute`, - ).to.exist; - } + assertAttributes(toolSpans, { + "gen_ai.operation.name": TOOL_OPERATION_NAME_PATTERN, + "gen_ai.tool.type": true, + "gen_ai.tool.name": true, + "gen_ai.tool.description": true, + }); }, }; @@ -835,7 +883,7 @@ export const checkBinaryRedaction: Check = { * Check attributes on handoff spans (agent-to-agent handoffs) * * Validates: - * - Handoff spans exist + * - gen_ai.operation.name matches HANDOFF_OPERATION_NAME_PATTERN * * Fails if no handoff spans are found. */ @@ -848,7 +896,9 @@ export const checkHandoffSpanAttributes: Check = { "Should have at least one handoff span", ).to.be.greaterThan(0); - // TODO: Add attribute validation once we know what attributes handoff spans should have + assertAttributes(handoffSpans, { + "gen_ai.operation.name": HANDOFF_OPERATION_NAME_PATTERN, + }); }, }; diff --git a/src/test-cases/utils.ts b/src/test-cases/utils.ts index 16852ba..fafecbd 100644 --- a/src/test-cases/utils.ts +++ b/src/test-cases/utils.ts @@ -38,11 +38,12 @@ export function skipIf(condition: boolean, reason: string): void { * Attribute schema for validation * - true: attribute must exist * - false: attribute must NOT exist + * - RegExp: must match the regular expression * - string with '*': must match pattern (glob-style) * - string/number: must equal exact value */ export type AttributeSchema = { - [key: string]: boolean | string | number; + [key: string]: boolean | string | number | RegExp; }; /** @@ -221,6 +222,7 @@ function matchPattern(value: string, pattern: string): boolean { * Schema format: * - true: attribute must exist (any value) * - false: attribute must NOT exist + * - RegExp: must match the regular expression * - string with '*': must match pattern (e.g., "gpt-4*" matches "gpt-4-turbo") * - string/number: must equal exact value * @@ -261,8 +263,23 @@ export function assertAttributes( `Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`, ); } + } else if (expected instanceof RegExp) { + // RegExp matching + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist for regex matching but is missing`, + ); + } else if (typeof actual !== "string") { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must be a string for regex matching but is: ${typeof actual}`, + ); + } else if (!expected.test(actual)) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match regex ${expected}`, + ); + } } else if (typeof expected === "string" && expected.includes("*")) { - // Pattern matching + // Pattern matching (glob-style) if (actual === undefined || actual === null) { errors.push( `Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`, From 56ebf3706796fc39c41e79a734ccb447583825b3 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 9 Feb 2026 14:20:23 +0100 Subject: [PATCH 2/3] feat: checking `span.name` and `span.description` in accordance with other attributes --- src/test-cases/checks.ts | 18 ++++ src/test-cases/utils.ts | 174 +++++++++++++++++++++++++-------------- 2 files changed, 128 insertions(+), 64 deletions(-) diff --git a/src/test-cases/checks.ts b/src/test-cases/checks.ts index 888c2ac..1252e88 100644 --- a/src/test-cases/checks.ts +++ b/src/test-cases/checks.ts @@ -179,6 +179,8 @@ export function checkAISpanCount( * Check attributes on chat/completion spans (LLM API calls) * * Validates: + * - span.description equals " " + * - span.name equals " " * - gen_ai.operation.name matches AI_CLIENT_OPERATION_NAME_PATTERN * - gen_ai.request.model matches expected model * - gen_ai.request.messages exists @@ -204,6 +206,10 @@ export const checkChatSpanAttributes: Check = { config.modelOverrides?.response || `${requestModel.replace("*", "")}*`; assertAttributes(chatSpans, { + "span.description": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, + "span.name": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, "gen_ai.operation.name": AI_CLIENT_OPERATION_NAME_PATTERN, "gen_ai.request.model": requestModel, "gen_ai.request.messages": true, @@ -219,6 +225,8 @@ export const checkChatSpanAttributes: Check = { * Check attributes on invoke_agent spans (agent invocations) * * Validates: + * - span.description equals " " + * - span.name equals " " * - gen_ai.operation.name matches AGENT_OPERATION_NAME_PATTERN * - gen_ai.agent.name exists * @@ -234,6 +242,10 @@ export const checkAgentSpanAttributes: Check = { ).to.be.greaterThan(0); assertAttributes(agentSpans, { + "span.description": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, + "span.name": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, "gen_ai.operation.name": AGENT_OPERATION_NAME_PATTERN, "gen_ai.agent.name": true, }); @@ -244,6 +256,8 @@ export const checkAgentSpanAttributes: Check = { * Check attributes on tool execution spans * * Validates: + * - span.description equals " " + * - span.name equals " " * - gen_ai.operation.name matches TOOL_OPERATION_NAME_PATTERN * - gen_ai.tool.type exists * - gen_ai.tool.name exists @@ -261,6 +275,10 @@ export const checkToolSpanAttributes: Check = { ).to.be.greaterThan(0); assertAttributes(toolSpans, { + "span.description": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.tool.name"]}`, + "span.name": (span) => + `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.tool.name"]}`, "gen_ai.operation.name": TOOL_OPERATION_NAME_PATTERN, "gen_ai.tool.type": true, "gen_ai.tool.name": true, diff --git a/src/test-cases/utils.ts b/src/test-cases/utils.ts index fafecbd..f1f07b3 100644 --- a/src/test-cases/utils.ts +++ b/src/test-cases/utils.ts @@ -34,6 +34,16 @@ export function skipIf(condition: boolean, reason: string): void { } } +/** + * Callable that receives a span and returns an expected value for validation. + * Use this to derive expected values dynamically from span attributes. + * + * @example + * // Check that description equals " " + * { "description": (span) => `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}` } + */ +export type AttributeSchemaFn = (span: CapturedSpan) => boolean | string | number | RegExp; + /** * Attribute schema for validation * - true: attribute must exist @@ -41,9 +51,10 @@ export function skipIf(condition: boolean, reason: string): void { * - RegExp: must match the regular expression * - string with '*': must match pattern (glob-style) * - string/number: must equal exact value + * - function(span): dynamically compute the expected value from the span */ export type AttributeSchema = { - [key: string]: boolean | string | number | RegExp; + [key: string]: boolean | string | number | RegExp | AttributeSchemaFn; }; /** @@ -216,15 +227,104 @@ function matchPattern(value: string, pattern: string): boolean { return regex.test(value); } +/** + * Resolve an attribute value from a span. + * Looks up in span.data first, then falls back to top-level span fields + * (e.g. "description", "op", "status"). + */ +function resolveAttribute(span: CapturedSpan, attrName: string): unknown { + // Check span.data first + if (span.data?.[attrName] !== undefined) { + return span.data[attrName]; + } + // Fall back to top-level span fields + if (attrName in span) { + return (span as Record)[attrName]; + } + return undefined; +} + +/** + * Validate a single attribute against an expected value, collecting errors. + */ +function validateAttribute( + actual: unknown, + expected: boolean | string | number | RegExp, + attrName: string, + spanIndex: number, + errors: string[], +): void { + if (expected === true) { + // Must exist + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist but is missing`, + ); + } + } else if (expected === false) { + // Must NOT exist + if (actual !== undefined && actual !== null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`, + ); + } + } else if (expected instanceof RegExp) { + // RegExp matching + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist for regex matching but is missing`, + ); + } else if (typeof actual !== "string") { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must be a string for regex matching but is: ${typeof actual}`, + ); + } else if (!expected.test(actual)) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match regex ${expected}`, + ); + } + } else if (typeof expected === "string" && expected.includes("*")) { + // Pattern matching (glob-style) + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`, + ); + } else if (typeof actual !== "string") { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must be a string for pattern matching but is: ${typeof actual}`, + ); + } else if (!matchPattern(actual, expected)) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match pattern '${expected}'`, + ); + } + } else { + // Exact value match + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is missing`, + ); + } else if (actual !== expected) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is '${actual}'`, + ); + } + } +} + /** * Assert attributes on spans based on schema * + * Attributes are resolved from span.data first, then from top-level span + * fields (e.g. "description", "op", "status"). + * * Schema format: * - true: attribute must exist (any value) * - false: attribute must NOT exist * - RegExp: must match the regular expression * - string with '*': must match pattern (e.g., "gpt-4*" matches "gpt-4-turbo") * - string/number: must equal exact value + * - function(span): dynamically compute the expected value from the span * * @param spans - List of spans to check (all spans must match schema) * @param schema - Attribute schema to validate against @@ -240,71 +340,17 @@ export function assertAttributes( const errors: string[] = []; spans.forEach((span, spanIndex) => { - if (!span.data) { - errors.push(`Span ${spanIndex}: Missing data field`); - return; - } - // Check each attribute in the schema - for (const [attrName, expected] of Object.entries(schema)) { - const actual = span.data[attrName]; + for (const [attrName, expectedOrFn] of Object.entries(schema)) { + const actual = resolveAttribute(span, attrName); - if (expected === true) { - // Must exist - if (actual === undefined || actual === null) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must exist but is missing`, - ); - } - } else if (expected === false) { - // Must NOT exist - if (actual !== undefined && actual !== null) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`, - ); - } - } else if (expected instanceof RegExp) { - // RegExp matching - if (actual === undefined || actual === null) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must exist for regex matching but is missing`, - ); - } else if (typeof actual !== "string") { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must be a string for regex matching but is: ${typeof actual}`, - ); - } else if (!expected.test(actual)) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match regex ${expected}`, - ); - } - } else if (typeof expected === "string" && expected.includes("*")) { - // Pattern matching (glob-style) - if (actual === undefined || actual === null) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`, - ); - } else if (typeof actual !== "string") { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must be a string for pattern matching but is: ${typeof actual}`, - ); - } else if (!matchPattern(actual, expected)) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match pattern '${expected}'`, - ); - } - } else { - // Exact value match - if (actual === undefined || actual === null) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is missing`, - ); - } else if (actual !== expected) { - errors.push( - `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is '${actual}'`, - ); - } - } + // Resolve callable to get the expected value for this span + const expected = + typeof expectedOrFn === "function" + ? expectedOrFn(span) + : expectedOrFn; + + validateAttribute(actual, expected, attrName, spanIndex, errors); } }); From ac7ee7ee8004d6585604517eb100e44f9619e1e9 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Tue, 10 Feb 2026 11:31:38 +0100 Subject: [PATCH 3/3] fix: remove span.name checks --- src/test-cases/checks.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/test-cases/checks.ts b/src/test-cases/checks.ts index 1252e88..61ed222 100644 --- a/src/test-cases/checks.ts +++ b/src/test-cases/checks.ts @@ -180,7 +180,6 @@ export function checkAISpanCount( * * Validates: * - span.description equals " " - * - span.name equals " " * - gen_ai.operation.name matches AI_CLIENT_OPERATION_NAME_PATTERN * - gen_ai.request.model matches expected model * - gen_ai.request.messages exists @@ -208,8 +207,6 @@ export const checkChatSpanAttributes: Check = { assertAttributes(chatSpans, { "span.description": (span) => `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, - "span.name": (span) => - `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, "gen_ai.operation.name": AI_CLIENT_OPERATION_NAME_PATTERN, "gen_ai.request.model": requestModel, "gen_ai.request.messages": true, @@ -226,7 +223,6 @@ export const checkChatSpanAttributes: Check = { * * Validates: * - span.description equals " " - * - span.name equals " " * - gen_ai.operation.name matches AGENT_OPERATION_NAME_PATTERN * - gen_ai.agent.name exists * @@ -244,8 +240,6 @@ export const checkAgentSpanAttributes: Check = { assertAttributes(agentSpans, { "span.description": (span) => `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, - "span.name": (span) => - `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`, "gen_ai.operation.name": AGENT_OPERATION_NAME_PATTERN, "gen_ai.agent.name": true, }); @@ -257,7 +251,6 @@ export const checkAgentSpanAttributes: Check = { * * Validates: * - span.description equals " " - * - span.name equals " " * - gen_ai.operation.name matches TOOL_OPERATION_NAME_PATTERN * - gen_ai.tool.type exists * - gen_ai.tool.name exists @@ -277,8 +270,6 @@ export const checkToolSpanAttributes: Check = { assertAttributes(toolSpans, { "span.description": (span) => `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.tool.name"]}`, - "span.name": (span) => - `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.tool.name"]}`, "gen_ai.operation.name": TOOL_OPERATION_NAME_PATTERN, "gen_ai.tool.type": true, "gen_ai.tool.name": true,