Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3,131 changes: 121 additions & 3,010 deletions package-lock.json

Large diffs are not rendered by default.

505 changes: 466 additions & 39 deletions packages/agents-a365-observability/README.md

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions packages/agents-a365-observability/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ export {
TenantDetails,
ToolCallDetails,
InvokeAgentDetails,
CallerDetails,
EnhancedAgentDetails,
ServiceEndpoint,
InferenceDetails,
InferenceOperationType,
InferenceResponse
Expand Down
92 changes: 76 additions & 16 deletions packages/agents-a365-observability/src/tracing/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,61 @@ export interface ToolCallDetails {

/** The type of the tool being executed */
toolType?: string;

/** The endpoint for the tool execution */
endpoint?: ServiceEndpoint;
}

/**
* Details about a caller
*/
export interface CallerDetails {
/** The unique identifier for the caller */
callerId?: string;

/** The user principal name (UPN) of the caller */
callerUpn?: string;

/** The display name of the caller */
callerName?: string;

/** The user ID of the caller */
callerUserId?: string;

/** The tenant ID of the caller */
tenantId?: string;
}

/**
* Enhanced agent details with additional properties
*/
export interface EnhancedAgentDetails extends AgentDetails {
/** The agent user ID (AUID) */
agentAUID?: string;

/** The agent user principal name (UPN) */
agentUPN?: string;

/** The agent blueprint/application ID */
agentBlueprintId?: string;

/** The tenant ID for the agent */
tenantId?: string;
}

/**
* Represents an endpoint for agent invocation
*/
export interface ServiceEndpoint {
/** The host address */
host: string;

/** The port number */
port?: number;

/** The protocol (e.g., http, https) */
protocol?: string;

}

/**
Expand All @@ -137,33 +192,38 @@ export interface ToolCallDetails {
export interface InvokeAgentDetails extends AgentDetails {
/** The request payload for the agent invocation */
request?: AgentRequest;
}

/** The endpoint for the agent invocation */
endpoint?: ServiceEndpoint;

/** Session ID for the invocation */
sessionId?: string;
}

/**
* Details for an LLM/AI model inference call
* Details for an inference call matching C# implementation
*/
export interface InferenceDetails {
/** The name/identifier of the model being used */
modelName: string;
/** The operation name/type for the inference */
operationName: InferenceOperationType;

/** The provider of the model (e.g., openai, azure, anthropic) */
provider?: string;
/** The model name/identifier */
model: string;

/** The specific model version or variant */
modelVersion?: string;
/** The provider name (e.g., openai, azure, anthropic) */
providerName?: string;

/** Temperature parameter for the model */
temperature?: number;
/** Number of input tokens used */
inputTokens?: number;

/** Maximum tokens to generate */
maxTokens?: number;
/** Number of output tokens generated */
outputTokens?: number;

/** Top-p parameter for the model */
topP?: number;
/** Array of finish reasons */
finishReasons?: string[];

/** Input prompt or messages to the model */
prompt?: string;
/** Response ID from the model provider */
responseId?: string;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ import { ToolCallDetails, AgentDetails, TenantDetails } from '../contracts';
import { OpenTelemetryConstants } from '../constants';

/**
* Provides OpenTelemetry tracing scope for AI tool execution operations
* Provides OpenTelemetry tracing scope for AI tool execution operations.
*/
export class ExecuteToolScope extends OpenTelemetryScope {
/**
* Creates and starts a new scope for tool execution tracing
* Creates and starts a new scope for tool execution tracing.
* @param details The tool call details
* @param agentDetails The agent details
* @param tenantDetails The tenant details
* @returns A new scope instance
* @returns A new ExecuteToolScope instance.
*/
public static start(details: ToolCallDetails, agentDetails: AgentDetails, tenantDetails: TenantDetails): ExecuteToolScope {
return new ExecuteToolScope(details, agentDetails, tenantDetails);
Expand All @@ -31,20 +31,31 @@ export class ExecuteToolScope extends OpenTelemetryScope {
tenantDetails
);

this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_NAME_KEY, details.toolName);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY, details.arguments);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_TYPE_KEY, details.toolType);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_CALL_ID_KEY, details.toolCallId);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_DESCRIPTION_KEY, details.description);
// Destructure the details object to match C# pattern
const { toolName, arguments: args, toolCallId, description, toolType, endpoint } = details;

this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_NAME_KEY, toolName);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY, args);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_TYPE_KEY, toolType);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_CALL_ID_KEY, toolCallId);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_DESCRIPTION_KEY, description);

// Set endpoint information if provided
if (endpoint) {
this.setTagMaybe(OpenTelemetryConstants.SERVER_ADDRESS_KEY, endpoint.host);

// Only record port if it is different from 443 (default HTTPS port)
if (endpoint.port && endpoint.port !== 443) {
this.setTagMaybe(OpenTelemetryConstants.SERVER_PORT_KEY, endpoint.port);
}
}
}

/**
* Records response information for telemetry tracking
* Records response information for telemetry tracking.
* @param response The tool execution response
*/
public recordResponse(response: string): void {
if (ExecuteToolScope.enableTelemetry) {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_TOOL_CALL_RESULT_KEY, response);
}
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_EVENT_CONTENT, response);
}
}
Comment thread
nikhilNava marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,22 @@
import { SpanKind } from '@opentelemetry/api';
import { OpenTelemetryScope } from './OpenTelemetryScope';
import { OpenTelemetryConstants } from '../constants';
import { InferenceDetails, InferenceResponse, InferenceOperationType, AgentDetails, TenantDetails } from '../contracts';
import {
InferenceDetails,
AgentDetails,
TenantDetails
} from '../contracts';

/**
* Provides tracing scope for LLM/AI model inference calls
* Provides OpenTelemetry tracing scope for generative AI inference operations.
*/
export class InferenceScope extends OpenTelemetryScope {
/**
* Creates and starts a new scope for inference tracing
* @param details The inference details
* Creates and starts a new scope for inference tracing.
* @param details The inference call details
* @param agentDetails The agent details
* @param tenantDetails The tenant details
* @returns A new scope instance
* @returns A new InferenceScope instance
*/
public static start(details: InferenceDetails, agentDetails: AgentDetails, tenantDetails: TenantDetails): InferenceScope {
return new InferenceScope(details, agentDetails, tenantDetails);
Expand All @@ -25,39 +29,71 @@ export class InferenceScope extends OpenTelemetryScope {
private constructor(details: InferenceDetails, agentDetails: AgentDetails, tenantDetails: TenantDetails) {
super(
SpanKind.CLIENT,
InferenceOperationType.CHAT,
`${InferenceOperationType.CHAT} ${details.modelName}`,
details.operationName.toString(),
`${details.operationName} ${details.model}`,
agentDetails,
tenantDetails
);

// Set model information
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY, details.modelName);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY, details.provider);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY, details.modelVersion);
// Set core inference information matching C# implementation
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY, details.operationName.toString());
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY, details.model);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY, details.providerName);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY, details.inputTokens?.toString());
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY, details.outputTokens?.toString());
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY, details.finishReasons?.join(','));
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_ID_KEY, details.responseId);
}

/**
* Records the input messages for telemetry tracking.
* @param messages Array of input messages
*/
public recordInputMessages(messages: string[]): void {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, messages.join(','));
}

/**
* Records the output messages for telemetry tracking.
* @param messages Array of output messages
*/
public recordOutputMessages(messages: string[]): void {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, messages.join(','));
}

// Set request parameters
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_TEMPERATURE_KEY, details.temperature);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_MAX_TOKENS_KEY, details.maxTokens);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_TOP_P_KEY, details.topP);
/**
* Records the number of input tokens for telemetry tracking.
* @param inputTokens Number of input tokens
*/
public recordInputTokens(inputTokens: number): void {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY, inputTokens.toString());
}

// Set prompt content if enabled
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, details.prompt);
/**
* Records the number of output tokens for telemetry tracking.
* @param outputTokens Number of output tokens
*/
public recordOutputTokens(outputTokens: number): void {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY, outputTokens.toString());
}

/**
* Records response information for inference telemetry tracking
* @param response The inference response details
* Records the response id for telemetry tracking.
* @param responseId The response ID
*/
public recordResponse(response: InferenceResponse): void {
if (InferenceScope.enableTelemetry) {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_EVENT_CONTENT, response.content);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_ID_KEY, response.responseId);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY, response.finishReason);
public recordResponseId(responseId: string): void {
if (responseId && responseId.trim()) {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_ID_KEY, responseId);
}
}

// Token usage metrics
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY, response.inputTokens);
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY, response.outputTokens);
/**
* Records the finish reasons for telemetry tracking.
* @param finishReasons Array of finish reasons
*/
public recordFinishReasons(finishReasons: string[]): void {
if (finishReasons && finishReasons.length > 0) {
this.setTagMaybe(OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY, finishReasons.join(','));
}
}

Expand Down
Loading
Loading