Skip to content

feat: checking span.name, span.description and gen_ai.operation.name #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
constantinius merged 3 commits into from
Feb 10, 2026
Merged

feat: checking span.name, span.description and gen_ai.operation.name #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b40ec74
feat: test `gen_ai.operation.name` to be of a certain pattern for the…
constantinius Feb 9, 2026
56ebf37
feat: checking `span.name` and `span.description` in accordance with …
constantinius Feb 9, 2026
ac7ee7e
fix: remove span.name checks
constantinius Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 84 additions & 25 deletions src/test-cases/checks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,63 @@ export interface Check {
fn: CheckFunction;
}

// =============================================================================
// Operation Name Patterns
// =============================================================================
// These patterns are derived from the Sentry backend logic that determines
// gen_ai.operation.type from gen_ai.operation.name.
//
// Reference (Rust code that determines operation type):
// - "agent" type: invoke_agent, create_agent, ai.run.*, ai.pipeline.*, ai.streamText, ai.generateText, ai.generateObject
// - "ai_client" type: *.doStream, *.doGenerate (the actual LLM API calls)
// - "tool" type: execute_tool, ai.toolCall.*
// - "handoff" type: handoff

/**
* Pattern for agent operation names (gen_ai.operation.name)
*
* Matches:
* - gen_ai.invoke_agent, invoke_agent
* - gen_ai.create_agent, create_agent
* - ai.run.generateText, ai.run.generateObject
* - ai.pipeline.generate_text, ai.pipeline.generate_object, ai.pipeline.stream_text, ai.pipeline.stream_object
* - ai.streamText (but NOT ai.streamText.doStream)
* - ai.generateText (but NOT ai.generateText.doGenerate)
* - ai.generateObject (but NOT ai.generateObject.doGenerate)
*/
export const AGENT_OPERATION_NAME_PATTERN =
/^(gen_ai\.)?(invoke_agent|create_agent)$|^ai\.run\.(generateText|generateObject)$|^ai\.pipeline\.(generate_text|generate_object|stream_text|stream_object)$|^ai\.(streamText|generateText|generateObject)(?!\.do)/;

/**
* Pattern for ai_client (chat/completion) operation names (gen_ai.operation.name)
*
* Matches:
* - ai.streamText.doStream.*
* - ai.generateText.doGenerate.*
* - ai.generateObject.doGenerate.*
* - chat, completion, generate (legacy)
*/
export const AI_CLIENT_OPERATION_NAME_PATTERN =
/^ai\.(streamText\.doStream|generateText\.doGenerate|generateObject\.doGenerate)|^(gen_ai\.)?(chat|completion|generate)/;

/**
* Pattern for tool operation names (gen_ai.operation.name)
*
* Matches:
* - gen_ai.execute_tool, execute_tool
* - ai.toolCall.*
*/
export const TOOL_OPERATION_NAME_PATTERN =
/^(gen_ai\.)?(execute_tool|tool|tool_call)$|^ai\.toolCall/;

/**
* Pattern for handoff operation names (gen_ai.operation.name)
*
* Matches:
* - gen_ai.handoff, handoff
*/
export const HANDOFF_OPERATION_NAME_PATTERN = /^(gen_ai\.)?handoff$/;

// =============================================================================
// Structure Checks
// =============================================================================
Expand Down Expand Up @@ -122,7 +179,8 @@ export function checkAISpanCount(
* Check attributes on chat/completion spans (LLM API calls)
*
* Validates:
* - gen_ai.operation.name exists
* - span.description equals "<gen_ai.operation.name> <gen_ai.request.model>"
* - gen_ai.operation.name matches AI_CLIENT_OPERATION_NAME_PATTERN
* - gen_ai.request.model matches expected model
* - gen_ai.request.messages exists
* - gen_ai.response.model matches expected pattern
Expand All @@ -147,7 +205,9 @@ export const checkChatSpanAttributes: Check = {
config.modelOverrides?.response || `${requestModel.replace("*", "")}*`;

assertAttributes(chatSpans, {
"gen_ai.operation.name": true,
"span.description": (span) =>
`${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`,
"gen_ai.operation.name": AI_CLIENT_OPERATION_NAME_PATTERN,
"gen_ai.request.model": requestModel,
"gen_ai.request.messages": true,
"gen_ai.response.model": responseModel,
Expand All @@ -162,6 +222,8 @@ export const checkChatSpanAttributes: Check = {
* Check attributes on invoke_agent spans (agent invocations)
*
* Validates:
* - span.description equals "<gen_ai.operation.name> <gen_ai.agent.name>"
* - gen_ai.operation.name matches AGENT_OPERATION_NAME_PATTERN
* - gen_ai.agent.name exists
*
* Fails if no agent spans are found.
Expand All @@ -175,20 +237,21 @@ export const checkAgentSpanAttributes: Check = {
"Should have at least one agent span",
).to.be.greaterThan(0);

// TODO: Add attribute validation once we know what attributes agent spans should have
for (const span of agentSpans) {
expect(
span.data?.["gen_ai.agent.name"],
`Agent span should have gen_ai.agent.name attribute`,
).to.exist;
}
assertAttributes(agentSpans, {
"span.description": (span) =>
`${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}`,
"gen_ai.operation.name": AGENT_OPERATION_NAME_PATTERN,
"gen_ai.agent.name": true,
});
},
};

/**
* Check attributes on tool execution spans
*
* Validates:
* - span.description equals "<gen_ai.operation.name> <gen_ai.tool.name>"
* - gen_ai.operation.name matches TOOL_OPERATION_NAME_PATTERN
* - gen_ai.tool.type exists
* - gen_ai.tool.name exists
* - gen_ai.tool.description exists
Expand All @@ -204,20 +267,14 @@ export const checkToolSpanAttributes: Check = {
"Should have at least one tool span",
).to.be.greaterThan(0);

for (const span of toolSpans) {
expect(
span.data?.["gen_ai.tool.type"],
`Tool span should have gen_ai.tool.type attribute`,
).to.exist;
expect(
span.data?.["gen_ai.tool.name"],
`Tool span should have gen_ai.tool.name attribute`,
).to.exist;
expect(
span.data?.["gen_ai.tool.description"],
`Tool span should have gen_ai.tool.description attribute`,
).to.exist;
}
assertAttributes(toolSpans, {
"span.description": (span) =>
`${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.tool.name"]}`,
"gen_ai.operation.name": TOOL_OPERATION_NAME_PATTERN,
"gen_ai.tool.type": true,
"gen_ai.tool.name": true,
"gen_ai.tool.description": true,
});
},
};

Expand Down Expand Up @@ -835,7 +892,7 @@ export const checkBinaryRedaction: Check = {
* Check attributes on handoff spans (agent-to-agent handoffs)
*
* Validates:
* - Handoff spans exist
* - gen_ai.operation.name matches HANDOFF_OPERATION_NAME_PATTERN
*
* Fails if no handoff spans are found.
*/
Expand All @@ -848,7 +905,9 @@ export const checkHandoffSpanAttributes: Check = {
"Should have at least one handoff span",
).to.be.greaterThan(0);

// TODO: Add attribute validation once we know what attributes handoff spans should have
assertAttributes(handoffSpans, {
"gen_ai.operation.name": HANDOFF_OPERATION_NAME_PATTERN,
});
},
};

Expand Down
161 changes: 112 additions & 49 deletions src/test-cases/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,27 @@ export function skipIf(condition: boolean, reason: string): void {
}
}

/**
* Callable that receives a span and returns an expected value for validation.
* Use this to derive expected values dynamically from span attributes.
*
* @example
* // Check that description equals "<operation.name> <model>"
* { "description": (span) => `${span.data?.["gen_ai.operation.name"]} ${span.data?.["gen_ai.request.model"]}` }
*/
export type AttributeSchemaFn = (span: CapturedSpan) => boolean | string | number | RegExp;

/**
* Attribute schema for validation
* - true: attribute must exist
* - false: attribute must NOT exist
* - RegExp: must match the regular expression
* - string with '*': must match pattern (glob-style)
* - string/number: must equal exact value
* - function(span): dynamically compute the expected value from the span
*/
export type AttributeSchema = {
[key: string]: boolean | string | number;
[key: string]: boolean | string | number | RegExp | AttributeSchemaFn;
};

/**
Expand Down Expand Up @@ -215,14 +227,104 @@ function matchPattern(value: string, pattern: string): boolean {
return regex.test(value);
}

/**
* Resolve an attribute value from a span.
* Looks up in span.data first, then falls back to top-level span fields
* (e.g. "description", "op", "status").
*/
function resolveAttribute(span: CapturedSpan, attrName: string): unknown {
// Check span.data first
if (span.data?.[attrName] !== undefined) {
return span.data[attrName];
}
// Fall back to top-level span fields
if (attrName in span) {
return (span as Record<string, unknown>)[attrName];
}
return undefined;
}

/**
* Validate a single attribute against an expected value, collecting errors.
*/
function validateAttribute(
actual: unknown,
expected: boolean | string | number | RegExp,
attrName: string,
spanIndex: number,
errors: string[],
): void {
if (expected === true) {
// Must exist
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must exist but is missing`,
);
}
} else if (expected === false) {
// Must NOT exist
if (actual !== undefined && actual !== null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`,
);
}
} else if (expected instanceof RegExp) {
// RegExp matching
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must exist for regex matching but is missing`,
);
} else if (typeof actual !== "string") {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must be a string for regex matching but is: ${typeof actual}`,
);
} else if (!expected.test(actual)) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match regex ${expected}`,
);
}
} else if (typeof expected === "string" && expected.includes("*")) {
// Pattern matching (glob-style)
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`,
);
} else if (typeof actual !== "string") {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must be a string for pattern matching but is: ${typeof actual}`,
);
} else if (!matchPattern(actual, expected)) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match pattern '${expected}'`,
);
}
} else {
// Exact value match
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is missing`,
);
} else if (actual !== expected) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is '${actual}'`,
);
}
}
}

/**
* Assert attributes on spans based on schema
*
* Attributes are resolved from span.data first, then from top-level span
* fields (e.g. "description", "op", "status").
*
* Schema format:
* - true: attribute must exist (any value)
* - false: attribute must NOT exist
* - RegExp: must match the regular expression
* - string with '*': must match pattern (e.g., "gpt-4*" matches "gpt-4-turbo")
* - string/number: must equal exact value
* - function(span): dynamically compute the expected value from the span
*
* @param spans - List of spans to check (all spans must match schema)
* @param schema - Attribute schema to validate against
Expand All @@ -238,56 +340,17 @@ export function assertAttributes(
const errors: string[] = [];

spans.forEach((span, spanIndex) => {
if (!span.data) {
errors.push(`Span ${spanIndex}: Missing data field`);
return;
}

// Check each attribute in the schema
for (const [attrName, expected] of Object.entries(schema)) {
const actual = span.data[attrName];
for (const [attrName, expectedOrFn] of Object.entries(schema)) {
const actual = resolveAttribute(span, attrName);

if (expected === true) {
// Must exist
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must exist but is missing`,
);
}
} else if (expected === false) {
// Must NOT exist
if (actual !== undefined && actual !== null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`,
);
}
} else if (typeof expected === "string" && expected.includes("*")) {
// Pattern matching
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`,
);
} else if (typeof actual !== "string") {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must be a string for pattern matching but is: ${typeof actual}`,
);
} else if (!matchPattern(actual, expected)) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match pattern '${expected}'`,
);
}
} else {
// Exact value match
if (actual === undefined || actual === null) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is missing`,
);
} else if (actual !== expected) {
errors.push(
`Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is '${actual}'`,
);
}
}
// Resolve callable to get the expected value for this span
const expected =
typeof expectedOrFn === "function"
? expectedOrFn(span)
: expectedOrFn;

validateAttribute(actual, expected, attrName, spanIndex, errors);
}
});

Expand Down
Loading