diff --git a/src/Observability/Runtime/DTOs/ApplyGuardrailData.cs b/src/Observability/Runtime/DTOs/ApplyGuardrailData.cs new file mode 100644 index 00000000..04ac5db3 --- /dev/null +++ b/src/Observability/Runtime/DTOs/ApplyGuardrailData.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using System; +using System.Collections.Generic; + +namespace Microsoft.Agents.A365.Observability.Runtime.DTOs +{ + /// + /// Encapsulates all telemetry data for an apply_guardrail operation. + /// + public class ApplyGuardrailData : BaseData + { + /// + /// Initializes a new instance of the class. + /// + /// The parent span ID for distributed tracing. + /// The telemetry attributes (tags). + /// Optional custom start time for the operation. + /// Optional custom end time for the operation. + /// Optional span ID for the operation. If not provided one will be created. + /// Optional span kind override. Defaults to null (unset). Use as appropriate. + /// Optional trace ID for distributed tracing. + public ApplyGuardrailData( + string parentSpanId, + IDictionary? attributes = null, + DateTimeOffset? startTime = null, + DateTimeOffset? endTime = null, + string? spanId = null, + string? spanKind = null, + string? traceId = null) + : base(attributes, startTime, endTime, spanId, parentSpanId, spanKind, traceId) + { } + + /// + /// Gets the name of the operation. + /// + public override string Name => OpenTelemetryConstants.OperationNames.ApplyGuardrail.ToString(); + } +} diff --git a/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs new file mode 100644 index 00000000..761bfbe7 --- /dev/null +++ b/src/Observability/Runtime/DTOs/Builders/ApplyGuardrailDataBuilder.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace Microsoft.Agents.A365.Observability.Runtime.DTOs.Builders +{ + /// + /// Builds an ApplyGuardrailData instance. + /// + public class ApplyGuardrailDataBuilder : BaseDataBuilder + { + /// + /// Builds complete data for an apply_guardrail operation. + /// + /// The details of the guardrail evaluation. + /// The details of the agent (includes tenant ID). + /// The conversation id. + /// The parent span ID for distributed tracing. + /// Optional custom start time for the operation. + /// Optional custom end time for the operation. + /// Optional span ID for the operation. + /// Optional channel information for the operation. + /// Optional details about the caller. + /// Optional dictionary of extra attributes. + /// Optional span kind override. + /// Optional trace ID for distributed tracing. + /// An ApplyGuardrailData object containing all telemetry data. + public static ApplyGuardrailData Build( + GuardrailDetails guardrailDetails, + AgentDetails agentDetails, + string conversationId, + string parentSpanId, + DateTimeOffset? startTime = null, + DateTimeOffset? endTime = null, + string? spanId = null, + Channel? channel = null, + CallerDetails? callerDetails = null, + IDictionary? extraAttributes = null, + string? spanKind = null, + string? traceId = null) + { + var attributes = BuildAttributes(guardrailDetails, agentDetails, conversationId, channel, callerDetails, extraAttributes); + + return new ApplyGuardrailData(parentSpanId, attributes, startTime, endTime, spanId, spanKind, traceId); + } + + private static Dictionary BuildAttributes( + GuardrailDetails guardrailDetails, + AgentDetails agentDetails, + string conversationId, + Channel? channel, + CallerDetails? callerDetails, + IDictionary? extraAttributes = null) + { + var attributes = new Dictionary(); + + // SDK attributes + AddSdkAttributes(attributes); + + // Operation name + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiOperationNameKey, OpenTelemetryConstants.ApplyGuardrailOperationName); + + AddAgentDetails(attributes, agentDetails); + + // Guardrail details + AddGuardrailDetails(attributes, guardrailDetails); + + // Conversation + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiConversationIdKey, conversationId); + + // Channel + AddChannelAttributes(attributes, channel); + + // Caller details + AddCallerDetails(attributes, callerDetails); + + // Extra attributes + AddExtraAttributes(attributes, extraAttributes); + + return attributes; + } + + private static void AddGuardrailDetails( + Dictionary attributes, + GuardrailDetails details) + { + // Required attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityDecisionTypeKey, details.DecisionType.ToString().ToLowerInvariant()); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityTargetTypeKey, details.TargetType); + + // Guardian attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiGuardianIdKey, details.GuardianId); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiGuardianNameKey, details.GuardianName); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiGuardianProviderNameKey, details.GuardianProviderName); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiGuardianVersionKey, details.GuardianVersion); + + // Target attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityTargetIdKey, details.TargetId); + + // Decision attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityDecisionReasonKey, details.DecisionReason); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityDecisionCodeKey, details.DecisionCode); + + // Policy attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityPolicyIdKey, details.PolicyId); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityPolicyNameKey, details.PolicyName); + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityPolicyVersionKey, details.PolicyVersion); + + // Content attributes + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityContentInputHashKey, details.ContentInputHash); + if (details.ContentModified.HasValue) + { + attributes[OpenTelemetryConstants.GenAiSecurityContentModifiedKey] = details.ContentModified.Value; + } + + // Correlation + AddIfNotNull(attributes, OpenTelemetryConstants.GenAiSecurityExternalEventIdKey, details.ExternalEventId); + } + + } +} diff --git a/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs index dc62235b..36bb3bab 100644 --- a/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/BaseDataBuilder.cs @@ -176,6 +176,16 @@ protected static void AddChannelAttributes(IDictionary attribut AddIfNotNull(attributes, OpenTelemetryConstants.ChannelLinkKey, channel.Link); } + /// + /// Adds telemetry SDK attributes (name, version, language) to the attributes dictionary. + /// + protected static void AddSdkAttributes(IDictionary attributes) + { + attributes[OpenTelemetryConstants.TelemetrySdkNameKey] = OpenTelemetryConstants.TelemetrySdkNameValue; + attributes[OpenTelemetryConstants.TelemetrySdkVersionKey] = OpenTelemetryConstants.TelemetrySdkVersionValue; + attributes[OpenTelemetryConstants.TelemetrySdkLanguageKey] = OpenTelemetryConstants.TelemetrySdkLanguageValue; + } + /// /// Adds a key-value pair to the dictionary if the value is not null. /// diff --git a/src/Observability/Runtime/DTOs/Builders/ExecuteInferenceDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/ExecuteInferenceDataBuilder.cs index 8e30c01f..a8898d08 100644 --- a/src/Observability/Runtime/DTOs/Builders/ExecuteInferenceDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/ExecuteInferenceDataBuilder.cs @@ -74,6 +74,9 @@ public static ExecuteInferenceData Build( { var attributes = new Dictionary(); + // SDK attributes + AddSdkAttributes(attributes); + // Agent details (includes tenant ID) AddAgentDetails(attributes, agentDetails); diff --git a/src/Observability/Runtime/DTOs/Builders/ExecuteToolDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/ExecuteToolDataBuilder.cs index 16566bf7..34f90618 100644 --- a/src/Observability/Runtime/DTOs/Builders/ExecuteToolDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/ExecuteToolDataBuilder.cs @@ -65,6 +65,9 @@ public static ExecuteToolData Build( { var attributes = new Dictionary(); + // SDK attributes + AddSdkAttributes(attributes); + // Operation name AddIfNotNull(attributes, GenAiOperationNameKey, ExecuteToolDataBuilder.ExecuteToolOperationName); diff --git a/src/Observability/Runtime/DTOs/Builders/InvokeAgentDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/InvokeAgentDataBuilder.cs index 72d58675..dcc0ecf1 100644 --- a/src/Observability/Runtime/DTOs/Builders/InvokeAgentDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/InvokeAgentDataBuilder.cs @@ -94,6 +94,9 @@ public static InvokeAgentData Build( { var attributes = new Dictionary(); + // SDK attributes + AddSdkAttributes(attributes); + // Operation name AddIfNotNull(attributes, GenAiOperationNameKey, InvokeAgentDataBuilder.InvokeAgentOperationName); diff --git a/src/Observability/Runtime/DTOs/Builders/OutputDataBuilder.cs b/src/Observability/Runtime/DTOs/Builders/OutputDataBuilder.cs index 59299989..85069ef8 100644 --- a/src/Observability/Runtime/DTOs/Builders/OutputDataBuilder.cs +++ b/src/Observability/Runtime/DTOs/Builders/OutputDataBuilder.cs @@ -59,6 +59,9 @@ public static OutputData Build( { var attributes = new Dictionary(); + // SDK attributes + AddSdkAttributes(attributes); + // Operation name AddIfNotNull(attributes, OpenTelemetryConstants.GenAiOperationNameKey, OutputMessagesOperationName); diff --git a/src/Observability/Runtime/Etw/A365EtwLogger.cs b/src/Observability/Runtime/Etw/A365EtwLogger.cs index 812aedb7..92f5363a 100644 --- a/src/Observability/Runtime/Etw/A365EtwLogger.cs +++ b/src/Observability/Runtime/Etw/A365EtwLogger.cs @@ -22,6 +22,8 @@ public class A365EtwLogger : IA365EtwLogger private static readonly EventId ExecuteToolEventId = new EventId(1003, ExecuteToolEventName); private const string OutputMessagesEventName = "OutputMessages"; private static readonly EventId OutputMessagesEventId = new EventId(1004, OutputMessagesEventName); + private const string ApplyGuardrailEventName = "ApplyGuardrail"; + private static readonly EventId ApplyGuardrailEventId = new EventId(1005, ApplyGuardrailEventName); /// /// Initializes a new instance of the class. @@ -179,6 +181,40 @@ public void LogOutput( ); } + /// + public void LogApplyGuardrail( + GuardrailDetails guardrailDetails, + AgentDetails agentDetails, + string conversationId, + string parentSpanId, + DateTimeOffset? startTime = null, + DateTimeOffset? endTime = null, + string? spanId = null, + Channel? channel = null, + CallerDetails? callerDetails = null, + string? traceId = null) + { + var data = ApplyGuardrailDataBuilder.Build( + guardrailDetails, + agentDetails, + conversationId, + parentSpanId, + startTime, + endTime, + spanId, + channel, + callerDetails: callerDetails, + traceId: traceId); + + logger.Log( + LogLevel.Information, + ApplyGuardrailEventId, + data.ToDictionary(), + null, + LogFormatter + ); + } + private static string LogFormatter(Dictionary data, Exception? ex) { return $"Name: {data["Name"]}, SpanId: {data["SpanId"]}, ParentSpanId: {data["ParentSpanId"]}, TraceId: {data["TraceId"]}"; diff --git a/src/Observability/Runtime/Etw/IA365EtwLogger.cs b/src/Observability/Runtime/Etw/IA365EtwLogger.cs index b96c5a35..5dee4308 100644 --- a/src/Observability/Runtime/Etw/IA365EtwLogger.cs +++ b/src/Observability/Runtime/Etw/IA365EtwLogger.cs @@ -120,5 +120,30 @@ public void LogOutput( string? spanId = null, string? parentSpanId = null, string? traceId = null); + + /// + /// Logs an apply_guardrail event. + /// + /// The details of the guardrail evaluation. + /// The details of the agent (includes tenant ID). + /// The required conversation ID. + /// The required parent span ID for tracing. + /// Optional start time of the guardrail evaluation. + /// Optional end time of the guardrail evaluation. + /// Optional span ID for tracing. + /// Optional channel information. + /// Optional details of the caller. + /// Optional trace ID for distributed tracing. + public void LogApplyGuardrail( + GuardrailDetails guardrailDetails, + AgentDetails agentDetails, + string conversationId, + string parentSpanId, + DateTimeOffset? startTime = null, + DateTimeOffset? endTime = null, + string? spanId = null, + Channel? channel = null, + CallerDetails? callerDetails = null, + string? traceId = null); } } diff --git a/src/Observability/Runtime/Tracing/Contracts/GuardrailDecisionType.cs b/src/Observability/Runtime/Tracing/Contracts/GuardrailDecisionType.cs new file mode 100644 index 00000000..aca5bf72 --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/GuardrailDecisionType.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.Serialization; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// The decision made by a security guardian during guardrail evaluation. + /// + [DataContract] + public enum GuardrailDecisionType + { + /// + /// Content or action is allowed to proceed. + /// + [EnumMember(Value = "allow")] + Allow, + + /// + /// Content or action is logged for review but allowed to proceed. + /// + [EnumMember(Value = "audit")] + Audit, + + /// + /// Content or action is denied/blocked. + /// + [EnumMember(Value = "deny")] + Deny, + + /// + /// Content was modified (e.g., redacted, sanitized, rewritten). + /// + [EnumMember(Value = "modify")] + Modify, + + /// + /// Content or action triggered a warning but is allowed to proceed. + /// + [EnumMember(Value = "warn")] + Warn + } +} diff --git a/src/Observability/Runtime/Tracing/Contracts/GuardrailDetails.cs b/src/Observability/Runtime/Tracing/Contracts/GuardrailDetails.cs new file mode 100644 index 00000000..6e10be14 --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/GuardrailDetails.cs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// Details of a guardrail evaluation for security operations tracing. + /// + public sealed class GuardrailDetails : IEquatable + { + /// + /// Initializes a new instance of the class. + /// + /// The type of content or action the guardrail is applied to (required). See for well-known values. + /// The decision made by the guardian (required). + /// Human-readable name of the guardian. + /// Unique identifier of the guardian. + /// Provider of the guardian service (e.g., azure.ai.content_safety). + /// Version of the guardian. + /// Identifier of the target being guarded. + /// Human-readable explanation for the decision. + /// Machine-readable decision code. + /// Identifier of the policy that triggered the decision. + /// Human-readable name of the policy. + /// Version of the policy. + /// Hash of the input content for forensic correlation. + /// Whether content was modified by the guardrail. + /// External correlation identifier for SIEM systems. + public GuardrailDetails( + string targetType, + GuardrailDecisionType decisionType, + string? guardianName = null, + string? guardianId = null, + string? guardianProviderName = null, + string? guardianVersion = null, + string? targetId = null, + string? decisionReason = null, + string? decisionCode = null, + string? policyId = null, + string? policyName = null, + string? policyVersion = null, + string? contentInputHash = null, + bool? contentModified = null, + string? externalEventId = null) + { + TargetType = targetType ?? throw new ArgumentNullException(nameof(targetType)); + DecisionType = decisionType; + GuardianName = guardianName; + GuardianId = guardianId; + GuardianProviderName = guardianProviderName; + GuardianVersion = guardianVersion; + TargetId = targetId; + DecisionReason = decisionReason; + DecisionCode = decisionCode; + PolicyId = policyId; + PolicyName = policyName; + PolicyVersion = policyVersion; + ContentInputHash = contentInputHash; + ContentModified = contentModified; + ExternalEventId = externalEventId; + } + + /// + /// Gets the type of content or action the guardrail is applied to. + /// + /// + public string TargetType { get; } + + /// + /// Gets the decision made by the security guardian. + /// + public GuardrailDecisionType DecisionType { get; } + + /// + /// Gets the human-readable name of the guardian. + /// + public string? GuardianName { get; } + + /// + /// Gets the unique identifier of the guardian. + /// + public string? GuardianId { get; } + + /// + /// Gets the provider of the guardian service. + /// + public string? GuardianProviderName { get; } + + /// + /// Gets the version of the guardian. + /// + public string? GuardianVersion { get; } + + /// + /// Gets the identifier of the target being guarded. + /// + public string? TargetId { get; } + + /// + /// Gets the human-readable explanation for the decision. + /// + public string? DecisionReason { get; } + + /// + /// Gets the machine-readable decision code. + /// + public string? DecisionCode { get; } + + /// + /// Gets the identifier of the policy that triggered the decision. + /// + public string? PolicyId { get; } + + /// + /// Gets the human-readable name of the policy. + /// + public string? PolicyName { get; } + + /// + /// Gets the version of the policy. + /// + public string? PolicyVersion { get; } + + /// + /// Gets the hash of the input content for forensic correlation. + /// + public string? ContentInputHash { get; } + + /// + /// Gets whether content was modified by the guardrail. + /// + public bool? ContentModified { get; } + + /// + /// Gets the external correlation identifier for SIEM systems. + /// + public string? ExternalEventId { get; } + + /// + public bool Equals(GuardrailDetails? other) + { + if (other is null) + { + return false; + } + + return string.Equals(TargetType, other.TargetType, StringComparison.Ordinal) && + DecisionType == other.DecisionType && + string.Equals(GuardianName, other.GuardianName, StringComparison.Ordinal) && + string.Equals(GuardianId, other.GuardianId, StringComparison.Ordinal) && + string.Equals(GuardianProviderName, other.GuardianProviderName, StringComparison.Ordinal) && + string.Equals(GuardianVersion, other.GuardianVersion, StringComparison.Ordinal) && + string.Equals(TargetId, other.TargetId, StringComparison.Ordinal) && + string.Equals(DecisionReason, other.DecisionReason, StringComparison.Ordinal) && + string.Equals(DecisionCode, other.DecisionCode, StringComparison.Ordinal) && + string.Equals(PolicyId, other.PolicyId, StringComparison.Ordinal) && + string.Equals(PolicyName, other.PolicyName, StringComparison.Ordinal) && + string.Equals(PolicyVersion, other.PolicyVersion, StringComparison.Ordinal) && + string.Equals(ContentInputHash, other.ContentInputHash, StringComparison.Ordinal) && + ContentModified == other.ContentModified && + string.Equals(ExternalEventId, other.ExternalEventId, StringComparison.Ordinal); + } + + /// + public override bool Equals(object? obj) + { + return Equals(obj as GuardrailDetails); + } + + /// + public override int GetHashCode() + { + unchecked + { + int hash = 17; + hash = (hash * 31) + (TargetType != null ? StringComparer.Ordinal.GetHashCode(TargetType) : 0); + hash = (hash * 31) + DecisionType.GetHashCode(); + hash = (hash * 31) + (GuardianName != null ? StringComparer.Ordinal.GetHashCode(GuardianName) : 0); + hash = (hash * 31) + (GuardianId != null ? StringComparer.Ordinal.GetHashCode(GuardianId) : 0); + hash = (hash * 31) + (GuardianProviderName != null ? StringComparer.Ordinal.GetHashCode(GuardianProviderName) : 0); + hash = (hash * 31) + (GuardianVersion != null ? StringComparer.Ordinal.GetHashCode(GuardianVersion) : 0); + hash = (hash * 31) + (TargetId != null ? StringComparer.Ordinal.GetHashCode(TargetId) : 0); + hash = (hash * 31) + (DecisionReason != null ? StringComparer.Ordinal.GetHashCode(DecisionReason) : 0); + hash = (hash * 31) + (DecisionCode != null ? StringComparer.Ordinal.GetHashCode(DecisionCode) : 0); + hash = (hash * 31) + (PolicyId != null ? StringComparer.Ordinal.GetHashCode(PolicyId) : 0); + hash = (hash * 31) + (PolicyName != null ? StringComparer.Ordinal.GetHashCode(PolicyName) : 0); + hash = (hash * 31) + (PolicyVersion != null ? StringComparer.Ordinal.GetHashCode(PolicyVersion) : 0); + hash = (hash * 31) + (ContentInputHash != null ? StringComparer.Ordinal.GetHashCode(ContentInputHash) : 0); + hash = (hash * 31) + (ContentModified?.GetHashCode() ?? 0); + hash = (hash * 31) + (ExternalEventId != null ? StringComparer.Ordinal.GetHashCode(ExternalEventId) : 0); + return hash; + } + } + } +} diff --git a/src/Observability/Runtime/Tracing/Contracts/GuardrailFinding.cs b/src/Observability/Runtime/Tracing/Contracts/GuardrailFinding.cs new file mode 100644 index 00000000..96cef690 --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/GuardrailFinding.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// Represents a single security finding detected during guardian evaluation. + /// Multiple findings may be emitted for a single guardrail span. + /// + public sealed class GuardrailFinding : IEquatable + { + /// + /// Initializes a new instance of the class. + /// + /// The category of security risk detected (required). + /// The severity level of the detected risk (required). + /// The decision type for this specific policy finding. + /// Identifier of the policy that triggered the finding. + /// Human-readable name of the triggered policy. + /// Version of the policy. + /// Numeric risk/confidence score (0.0 to 1.0). + /// Non-content metadata about the detected risk (MUST NOT contain PII). + public GuardrailFinding( + string riskCategory, + string riskSeverity, + string? policyDecisionType = null, + string? policyId = null, + string? policyName = null, + string? policyVersion = null, + double? riskScore = null, + string[]? riskMetadata = null) + { + RiskCategory = riskCategory ?? throw new ArgumentNullException(nameof(riskCategory)); + RiskSeverity = riskSeverity ?? throw new ArgumentNullException(nameof(riskSeverity)); + PolicyDecisionType = policyDecisionType; + PolicyId = policyId; + PolicyName = policyName; + PolicyVersion = policyVersion; + RiskScore = riskScore; + RiskMetadata = riskMetadata; + } + + /// + /// Gets the category of security risk detected. + /// + /// + /// Free-form field aligned with OWASP LLM Top 10 2025. + /// Common values: prompt_injection, sensitive_info_disclosure, jailbreak, toxicity, pii. + /// + public string RiskCategory { get; } + + /// + /// Gets the severity level of the detected risk. + /// + /// + public string RiskSeverity { get; } + + /// + /// Gets the decision type for this specific policy finding. + /// + public string? PolicyDecisionType { get; } + + /// + /// Gets the identifier of the policy that triggered the finding. + /// + public string? PolicyId { get; } + + /// + /// Gets the human-readable name of the triggered policy. + /// + public string? PolicyName { get; } + + /// + /// Gets the version of the policy. + /// + public string? PolicyVersion { get; } + + /// + /// Gets the numeric risk/confidence score (0.0 to 1.0). + /// + public double? RiskScore { get; } + + /// + /// Gets non-content metadata about the detected risk. + /// + /// + /// This MUST NOT contain sensitive user content, PII, or other high-risk data. + /// Example values: "field:bcc", "pattern:ssn", "count:3", "position:input[0].content". + /// + public string[]? RiskMetadata { get; } + + /// + public bool Equals(GuardrailFinding? other) + { + if (other is null) + { + return false; + } + + return string.Equals(RiskCategory, other.RiskCategory, StringComparison.Ordinal) && + string.Equals(RiskSeverity, other.RiskSeverity, StringComparison.Ordinal) && + string.Equals(PolicyDecisionType, other.PolicyDecisionType, StringComparison.Ordinal) && + string.Equals(PolicyId, other.PolicyId, StringComparison.Ordinal) && + string.Equals(PolicyName, other.PolicyName, StringComparison.Ordinal) && + string.Equals(PolicyVersion, other.PolicyVersion, StringComparison.Ordinal) && + RiskScore == other.RiskScore; + } + + /// + public override bool Equals(object? obj) + { + return Equals(obj as GuardrailFinding); + } + + /// + public override int GetHashCode() + { + unchecked + { + int hash = 17; + hash = (hash * 31) + (RiskCategory != null ? StringComparer.Ordinal.GetHashCode(RiskCategory) : 0); + hash = (hash * 31) + (RiskSeverity != null ? StringComparer.Ordinal.GetHashCode(RiskSeverity) : 0); + hash = (hash * 31) + (PolicyDecisionType != null ? StringComparer.Ordinal.GetHashCode(PolicyDecisionType) : 0); + hash = (hash * 31) + (PolicyId != null ? StringComparer.Ordinal.GetHashCode(PolicyId) : 0); + hash = (hash * 31) + (PolicyName != null ? StringComparer.Ordinal.GetHashCode(PolicyName) : 0); + hash = (hash * 31) + (PolicyVersion != null ? StringComparer.Ordinal.GetHashCode(PolicyVersion) : 0); + hash = (hash * 31) + (RiskScore?.GetHashCode() ?? 0); + return hash; + } + } + } +} diff --git a/src/Observability/Runtime/Tracing/Contracts/GuardrailRiskSeverity.cs b/src/Observability/Runtime/Tracing/Contracts/GuardrailRiskSeverity.cs new file mode 100644 index 00000000..9576c2a9 --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/GuardrailRiskSeverity.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// Well-known severity levels for security risks detected by guardrails. + /// + public static class GuardrailRiskSeverity + { + /// + /// No risk detected. + /// + public const string None = "none"; + + /// + /// Low severity risk. + /// + public const string Low = "low"; + + /// + /// Medium severity risk. + /// + public const string Medium = "medium"; + + /// + /// High severity risk. + /// + public const string High = "high"; + + /// + /// Critical severity risk requiring immediate action. + /// + public const string Critical = "critical"; + } +} diff --git a/src/Observability/Runtime/Tracing/Contracts/GuardrailTargetType.cs b/src/Observability/Runtime/Tracing/Contracts/GuardrailTargetType.cs new file mode 100644 index 00000000..7da15ca9 --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/GuardrailTargetType.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// Well-known values for the type of content or action a guardrail is applied to. + /// + /// + /// This is a free-form field per the OpenTelemetry semantic conventions. + /// These static members provide discoverability for common values, but custom strings + /// are accepted via implicit conversion (e.g. GuardrailTargetType myType = "custom_target";). + /// + public readonly struct GuardrailTargetType : IEquatable + { + /// + /// Input to a language model. + /// + public static readonly GuardrailTargetType LlmInput = new GuardrailTargetType("llm_input"); + + /// + /// Output from a language model. + /// + public static readonly GuardrailTargetType LlmOutput = new GuardrailTargetType("llm_output"); + + /// + /// A tool call action. + /// + public static readonly GuardrailTargetType ToolCall = new GuardrailTargetType("tool_call"); + + /// + /// A tool definition. + /// + public static readonly GuardrailTargetType ToolDefinition = new GuardrailTargetType("tool_definition"); + + /// + /// A memory store operation. + /// + public static readonly GuardrailTargetType MemoryStore = new GuardrailTargetType("memory_store"); + + /// + /// A memory retrieval operation. + /// + public static readonly GuardrailTargetType MemoryRetrieve = new GuardrailTargetType("memory_retrieve"); + + /// + /// A knowledge query. + /// + public static readonly GuardrailTargetType KnowledgeQuery = new GuardrailTargetType("knowledge_query"); + + /// + /// A knowledge retrieval result. + /// + public static readonly GuardrailTargetType KnowledgeResult = new GuardrailTargetType("knowledge_result"); + + /// + /// A message. + /// + public static readonly GuardrailTargetType Message = new GuardrailTargetType("message"); + + /// + /// Gets the string value of this target type. + /// + public string Value { get; } + + /// + /// Initializes a new instance of the struct. + /// + /// The target type string value. + public GuardrailTargetType(string value) + { + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// Implicitly converts a string to a . + /// + public static implicit operator GuardrailTargetType(string value) => new GuardrailTargetType(value); + + /// + /// Implicitly converts a to its string value. + /// + public static implicit operator string(GuardrailTargetType type) => type.Value; + + /// + public bool Equals(GuardrailTargetType other) => string.Equals(Value, other.Value, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is GuardrailTargetType other && Equals(other); + + /// + public override int GetHashCode() => Value != null ? StringComparer.Ordinal.GetHashCode(Value) : 0; + + /// + public override string ToString() => Value; + + /// + /// Equality operator. + /// + public static bool operator ==(GuardrailTargetType left, GuardrailTargetType right) => left.Equals(right); + + /// + /// Inequality operator. + /// + public static bool operator !=(GuardrailTargetType left, GuardrailTargetType right) => !left.Equals(right); + } +} diff --git a/src/Observability/Runtime/Tracing/Scopes/ApplyGuardrailScope.cs b/src/Observability/Runtime/Tracing/Scopes/ApplyGuardrailScope.cs new file mode 100644 index 00000000..03dd008d --- /dev/null +++ b/src/Observability/Runtime/Tracing/Scopes/ApplyGuardrailScope.cs @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes +{ + /// + /// Provides OpenTelemetry tracing scope for security guardrail evaluation operations. + /// + /// + /// + /// Describes a security guardian evaluation. Multiple guardian spans MAY exist under a single + /// operation span if multiple guardians are chained. + /// + /// + /// Guardian spans SHOULD be children of the operation span they are protecting + /// (e.g., inference or execute_tool spans). + /// + /// + public sealed class ApplyGuardrailScope : OpenTelemetryScope + { + + /// + /// Creates and starts a new scope for guardrail evaluation tracing. + /// + /// Details of the guardrail evaluation (target, decision, guardian info, policy). + /// Information about the agent being guarded. + /// Optional request details for conversation context. + /// Optional human user details. + /// Optional span configuration (parent context, timing, kind, span links). + /// A new ApplyGuardrailScope instance. + /// + /// + /// Certification Requirements: The following parameters must be set for the agent to pass certification requirements: + /// + /// + /// + /// + /// + /// + public static ApplyGuardrailScope Start( + GuardrailDetails details, + AgentDetails agentDetails, + Request? request = null, + UserDetails? userDetails = null, + SpanDetails? spanDetails = null) => new ApplyGuardrailScope(details, agentDetails, request, userDetails, spanDetails); + + private ApplyGuardrailScope( + GuardrailDetails details, + AgentDetails agentDetails, + Request? request, + UserDetails? userDetails, + SpanDetails? spanDetails) + : base( + operationName: OpenTelemetryConstants.ApplyGuardrailOperationName, + activityName: BuildActivityName(details), + agentDetails: agentDetails, + spanDetails: new SpanDetails(spanDetails?.SpanKind ?? ActivityKind.Internal, spanDetails?.ParentContext, spanDetails?.StartTime, spanDetails?.EndTime, spanDetails?.SpanLinks), + userDetails: userDetails) + { + // Required attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityDecisionTypeKey, details.DecisionType.ToString().ToLowerInvariant()); + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityTargetTypeKey, details.TargetType); + + // Guardian attributes + SetTagMaybe(OpenTelemetryConstants.GenAiGuardianIdKey, details.GuardianId); + SetTagMaybe(OpenTelemetryConstants.GenAiGuardianNameKey, details.GuardianName); + SetTagMaybe(OpenTelemetryConstants.GenAiGuardianProviderNameKey, details.GuardianProviderName); + SetTagMaybe(OpenTelemetryConstants.GenAiGuardianVersionKey, details.GuardianVersion); + + // Target attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityTargetIdKey, details.TargetId); + + // Decision attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityDecisionReasonKey, details.DecisionReason); + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityDecisionCodeKey, details.DecisionCode); + + // Policy attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityPolicyIdKey, details.PolicyId); + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityPolicyNameKey, details.PolicyName); + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityPolicyVersionKey, details.PolicyVersion); + + // Content attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityContentInputHashKey, details.ContentInputHash); + if (details.ContentModified.HasValue) + { + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityContentModifiedKey, details.ContentModified.Value); + } + + // Correlation attributes + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityExternalEventIdKey, details.ExternalEventId); + + // Request context + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityContentInputValueKey, request?.Content); + SetTagMaybe(OpenTelemetryConstants.GenAiConversationIdKey, request?.ConversationId); + if (request?.Channel != null) + { + SetTagMaybe(OpenTelemetryConstants.ChannelNameKey, request.Channel.Name); + SetTagMaybe(OpenTelemetryConstants.ChannelLinkKey, request.Channel.Link); + } + } + + /// + /// Records an updated decision on the guardrail span. + /// Use this when the guardrail decision is determined after span creation. + /// + /// The decision type made by the guardian. + /// Optional human-readable explanation for the decision. + public void RecordDecision(GuardrailDecisionType decisionType, string? reason = null) + { + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityDecisionTypeKey, decisionType.ToString().ToLowerInvariant()); + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityDecisionReasonKey, reason); + } + + /// + /// Records the output content value for the guardrail evaluation (opt-in). + /// + /// The output content after guardrail processing. + public void RecordContentOutput(string outputValue) + { + SetTagMaybe(OpenTelemetryConstants.GenAiSecurityContentOutputValueKey, outputValue); + } + + /// + /// Records a security finding event on the current span. + /// Multiple findings may be recorded per guardrail evaluation. + /// + /// The security finding to record. + /// Thrown when is null. + public void RecordFinding(GuardrailFinding finding) + { + if (finding == null) + { + throw new ArgumentNullException(nameof(finding)); + } + + var tags = new ActivityTagsCollection + { + { OpenTelemetryConstants.GenAiSecurityRiskCategoryKey, finding.RiskCategory }, + { OpenTelemetryConstants.GenAiSecurityRiskSeverityKey, finding.RiskSeverity } + }; + + if (finding.PolicyDecisionType != null) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey, finding.PolicyDecisionType); + } + + if (finding.PolicyId != null) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityPolicyIdKey, finding.PolicyId); + } + + if (finding.PolicyName != null) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityPolicyNameKey, finding.PolicyName); + } + + if (finding.PolicyVersion != null) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityPolicyVersionKey, finding.PolicyVersion); + } + + if (finding.RiskScore.HasValue) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityRiskScoreKey, finding.RiskScore.Value); + } + + if (finding.RiskMetadata != null) + { + tags.Add(OpenTelemetryConstants.GenAiSecurityRiskMetadataKey, finding.RiskMetadata); + } + + var activityEvent = new ActivityEvent(OpenTelemetryConstants.GenAiSecurityFindingEventName, tags: tags); + AddEvent(activityEvent); + } + + private static string BuildActivityName(GuardrailDetails details) + { + if (!string.IsNullOrWhiteSpace(details.GuardianName)) + { + return $"{OpenTelemetryConstants.ApplyGuardrailOperationName} {details.GuardianName} {details.TargetType}"; + } + + return $"{OpenTelemetryConstants.ApplyGuardrailOperationName} {details.TargetType}"; + } + + } +} diff --git a/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryConstants.cs b/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryConstants.cs index 1d233d82..c51053f8 100644 --- a/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryConstants.cs +++ b/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryConstants.cs @@ -49,9 +49,17 @@ public enum OperationNames ExecuteTool, [EnumMember(Value = "OutputMessages")] - OutputMessages + OutputMessages, + + [EnumMember(Value = "ApplyGuardrail")] + ApplyGuardrail } + /// + /// The operation name value for apply_guardrail spans. + /// + public const string ApplyGuardrailOperationName = "apply_guardrail"; + // Channel dimensions (renamed from gen_ai.channel.* to microsoft.channel.*) public const string ChannelNameKey = "microsoft.channel.name"; public const string ChannelLinkKey = "microsoft.channel.link"; @@ -161,6 +169,123 @@ public enum OperationNames public const string GenAiAgentThoughtProcessKey = "microsoft.a365.agent.thought.process"; #endregion + + #region guardrail keys + /// + /// The guardian identifier key. + /// + public const string GenAiGuardianIdKey = "microsoft.guardian.id"; + + /// + /// The guardian name key. + /// + public const string GenAiGuardianNameKey = "microsoft.guardian.name"; + + /// + /// The guardian provider name key. + /// + public const string GenAiGuardianProviderNameKey = "microsoft.guardian.provider.name"; + + /// + /// The guardian version key. + /// + public const string GenAiGuardianVersionKey = "microsoft.guardian.version"; + + /// + /// The security decision type key. + /// + public const string GenAiSecurityDecisionTypeKey = "microsoft.security.decision.type"; + + /// + /// The security decision reason key. + /// + public const string GenAiSecurityDecisionReasonKey = "microsoft.security.decision.reason"; + + /// + /// The security decision code key. + /// + public const string GenAiSecurityDecisionCodeKey = "microsoft.security.decision.code"; + + /// + /// The security target type key. + /// + public const string GenAiSecurityTargetTypeKey = "microsoft.security.target.type"; + + /// + /// The security target identifier key. + /// + public const string GenAiSecurityTargetIdKey = "microsoft.security.target.id"; + + /// + /// The security policy identifier key. + /// + public const string GenAiSecurityPolicyIdKey = "microsoft.security.policy.id"; + + /// + /// The security policy name key. + /// + public const string GenAiSecurityPolicyNameKey = "microsoft.security.policy.name"; + + /// + /// The security policy version key. + /// + public const string GenAiSecurityPolicyVersionKey = "microsoft.security.policy.version"; + + /// + /// The security content input hash key. + /// + public const string GenAiSecurityContentInputHashKey = "microsoft.security.content.input.hash"; + + /// + /// The security content modified key. + /// + public const string GenAiSecurityContentModifiedKey = "microsoft.security.content.modified"; + + /// + /// The security external event identifier key for SIEM correlation. + /// + public const string GenAiSecurityExternalEventIdKey = "microsoft.security.external_event_id"; + + /// + /// The security content input value key (opt-in). + /// + public const string GenAiSecurityContentInputValueKey = "microsoft.security.content.input.value"; + + /// + /// The security content output value key (opt-in). + /// + public const string GenAiSecurityContentOutputValueKey = "microsoft.security.content.output.value"; + + /// + /// The security finding event name. + /// + public const string GenAiSecurityFindingEventName = "microsoft.security.finding"; + + /// + /// The security risk category key. + /// + public const string GenAiSecurityRiskCategoryKey = "microsoft.security.risk.category"; + + /// + /// The security risk severity key. + /// + public const string GenAiSecurityRiskSeverityKey = "microsoft.security.risk.severity"; + + /// + /// The security risk score key. + /// + public const string GenAiSecurityRiskScoreKey = "microsoft.security.risk.score"; + + /// + /// The security risk metadata key. + /// + public const string GenAiSecurityRiskMetadataKey = "microsoft.security.risk.metadata"; + + /// + /// The security policy decision type key (per-finding decision). + /// + public const string GenAiSecurityPolicyDecisionTypeKey = "microsoft.security.policy.decision.type"; + #endregion } #pragma warning restore CS1591 } diff --git a/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryScope.cs b/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryScope.cs index 34b1d926..cffaa376 100644 --- a/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryScope.cs +++ b/src/Observability/Runtime/Tracing/Scopes/OpenTelemetryScope.cs @@ -237,6 +237,15 @@ protected void AddBaggage(string key, string value) activity?.AddBaggage(key, value); } + /// + /// Adds an event to the current activity. + /// + /// The event to add. + protected void AddEvent(ActivityEvent activityEvent) + { + activity?.AddEvent(activityEvent); + } + /// /// Gets the for this scope's span. /// diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs index 02a62600..5eb3a573 100644 --- a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs @@ -7,6 +7,7 @@ using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using System.Linq; using System.Net; using System.Text.Json; @@ -517,6 +518,158 @@ public async Task Exporter_Truncates_Scope() /// This test ensures the exporter registry correctly implements singleton behavior /// to avoid data duplication when the same exporter type is configured multiple times. /// + [TestMethod] + public void AddTracing_And_ApplyGuardrailScope_ExporterMakesExpectedRequest() + { + // Arrange + var expectedAgentDetails = new AgentDetails( + agentId: Guid.NewGuid().ToString(), + agentName: "Guardrail Agent", + agentDescription: "Agent for guardrail testing.", + agenticUserId: Guid.NewGuid().ToString(), + agenticUserEmail: "guardrailagent@ztaittest12.onmicrosoft.com", + agentBlueprintId: Guid.NewGuid().ToString(), + tenantId: Guid.NewGuid().ToString(), + agentType: AgentType.EntraEmbodied); + + var guardrailDetails = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny, + guardianId: "azure-content-safety-001", + guardianName: "Azure Content Safety", + guardianProviderName: "Azure", + guardianVersion: "2.0.0", + targetId: "msg-12345", + decisionReason: "Content violates hate speech policy", + decisionCode: "HATE_SPEECH_001", + policyId: "policy-abc", + policyName: "Content Safety Policy", + policyVersion: "1.2.0", + contentInputHash: "sha256:abc123def456", + contentModified: false, + externalEventId: "ext-event-789"); + + var expectedUserDetails = new UserDetails( + userId: "guardrail-caller-123", + userName: "Guardrail Caller", + userEmail: "guardrail-caller@ztaitest12.onmicrosoft.com", + userClientIP: IPAddress.Parse("192.168.1.42")); + + var expectedRequest = new Request( + content: "Check this content for safety", + channel: new Channel( + name: "msteams", + link: "https://guardrail-channel.link")); + + // Use ActivityListener to capture the activity directly + System.Diagnostics.Activity? capturedActivity = null; + using var listener = new System.Diagnostics.ActivityListener(); + listener.ShouldListenTo = source => source.Name == OpenTelemetryConstants.SourceName; + listener.Sample = (ref System.Diagnostics.ActivityCreationOptions _) => + System.Diagnostics.ActivitySamplingResult.AllDataAndRecorded; + listener.ActivityStarted = a => capturedActivity = a; + System.Diagnostics.ActivitySource.AddActivityListener(listener); + + // Act + using (var scope = ApplyGuardrailScope.Start( + details: guardrailDetails, + agentDetails: expectedAgentDetails, + request: expectedRequest, + userDetails: expectedUserDetails)) + { + scope.RecordDecision(GuardrailDecisionType.Deny); + scope.RecordContentOutput("sanitized-hash-output"); + scope.RecordFinding(new GuardrailFinding( + riskCategory: "hate_speech", + riskSeverity: GuardrailRiskSeverity.High, + policyDecisionType: "deny", + policyId: "policy-abc", + riskScore: 0.95, + riskMetadata: new[] { "{\"category\":\"hate\",\"confidence\":0.95}" })); + } + + // Assert activity was captured + capturedActivity.Should().NotBeNull("ApplyGuardrailScope should create an activity"); + + // Format activity as OTLP-like JSON for inspection + var spanJson = new + { + name = capturedActivity!.DisplayName, + traceId = capturedActivity.TraceId.ToHexString(), + spanId = capturedActivity.SpanId.ToHexString(), + parentSpanId = capturedActivity.ParentSpanId.ToHexString(), + kind = capturedActivity.Kind.ToString(), + startTimeUnixNano = capturedActivity.StartTimeUtc.Ticks, + endTimeUnixNano = (capturedActivity.StartTimeUtc + capturedActivity.Duration).Ticks, + attributes = capturedActivity.TagObjects.ToDictionary(t => t.Key, t => t.Value), + events = capturedActivity.Events.Select(e => new + { + name = e.Name, + timeUnixNano = e.Timestamp.Ticks, + attributes = e.Tags.ToDictionary(t => t.Key, t => t.Value) + }).ToArray() + }; + + var prettyJson = JsonSerializer.Serialize(spanJson, new JsonSerializerOptions { WriteIndented = true }); + Console.WriteLine("=== ApplyGuardrail Span JSON ==="); + Console.WriteLine(prettyJson); + Console.WriteLine("=== End ApplyGuardrail Span JSON ==="); + + // Verify span name + capturedActivity.DisplayName.Should().Be("apply_guardrail Azure Content Safety llm_input"); + + // Verify operation name + capturedActivity.GetTagItem("gen_ai.operation.name").Should().Be("apply_guardrail"); + + // Verify agent details + capturedActivity.GetTagItem("gen_ai.agent.id").Should().Be(expectedAgentDetails.AgentId); + capturedActivity.GetTagItem("gen_ai.agent.name").Should().Be(expectedAgentDetails.AgentName); + capturedActivity.GetTagItem("gen_ai.agent.description").Should().Be(expectedAgentDetails.AgentDescription); + capturedActivity.GetTagItem("microsoft.agent.user.id").Should().Be(expectedAgentDetails.AgenticUserId); + capturedActivity.GetTagItem("microsoft.agent.user.email").Should().Be(expectedAgentDetails.AgenticUserEmail); + capturedActivity.GetTagItem("microsoft.a365.agent.blueprint.id").Should().Be(expectedAgentDetails.AgentBlueprintId); + capturedActivity.GetTagItem("microsoft.tenant.id").Should().Be(expectedAgentDetails.TenantId); + + // Verify guardrail-specific attributes + capturedActivity.GetTagItem("microsoft.security.decision.type").Should().Be("deny"); + capturedActivity.GetTagItem("microsoft.security.target.type").Should().Be("llm_input"); + capturedActivity.GetTagItem("microsoft.guardian.id").Should().Be("azure-content-safety-001"); + capturedActivity.GetTagItem("microsoft.guardian.name").Should().Be("Azure Content Safety"); + capturedActivity.GetTagItem("microsoft.guardian.provider.name").Should().Be("Azure"); + capturedActivity.GetTagItem("microsoft.guardian.version").Should().Be("2.0.0"); + capturedActivity.GetTagItem("microsoft.security.target.id").Should().Be("msg-12345"); + capturedActivity.GetTagItem("microsoft.security.decision.reason").Should().Be("Content violates hate speech policy"); + capturedActivity.GetTagItem("microsoft.security.decision.code").Should().Be("HATE_SPEECH_001"); + capturedActivity.GetTagItem("microsoft.security.policy.id").Should().Be("policy-abc"); + capturedActivity.GetTagItem("microsoft.security.policy.name").Should().Be("Content Safety Policy"); + capturedActivity.GetTagItem("microsoft.security.policy.version").Should().Be("1.2.0"); + capturedActivity.GetTagItem("microsoft.security.content.input.hash").Should().Be("sha256:abc123def456"); + capturedActivity.GetTagItem("microsoft.security.content.output.value").Should().Be("sanitized-hash-output"); + capturedActivity.GetTagItem("microsoft.security.content.modified").Should().Be(false); + capturedActivity.GetTagItem("microsoft.security.external_event_id").Should().Be("ext-event-789"); + + // Verify caller details + capturedActivity.GetTagItem("user.id").Should().Be(expectedUserDetails.UserId); + capturedActivity.GetTagItem("user.email").Should().Be(expectedUserDetails.UserEmail); + capturedActivity.GetTagItem("user.name").Should().Be(expectedUserDetails.UserName); + capturedActivity.GetTagItem("client.address").Should().Be(expectedUserDetails.UserClientIP?.ToString()); + + // Verify channel + capturedActivity.GetTagItem("microsoft.channel.name").Should().Be(expectedRequest.Channel?.Name); + capturedActivity.GetTagItem("microsoft.channel.link").Should().Be(expectedRequest.Channel?.Link); + + // Verify events (finding) + capturedActivity.Events.Should().HaveCount(1); + var findingEvent = capturedActivity.Events.First(); + findingEvent.Name.Should().Be("microsoft.security.finding"); + var findingTags = findingEvent.Tags.ToDictionary(t => t.Key, t => t.Value); + findingTags["microsoft.security.risk.category"].Should().Be("hate_speech"); + findingTags["microsoft.security.risk.severity"].Should().Be(GuardrailRiskSeverity.High); + findingTags["microsoft.security.policy.decision.type"].Should().Be("deny"); + findingTags["microsoft.security.policy.id"].Should().Be("policy-abc"); + findingTags["microsoft.security.risk.score"].Should().Be(0.95); + } + [TestMethod] public async Task AddTracing_MultipleInvocations_NoDuplicateExports() { diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/ApplyGuardrailScopeTest.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/ApplyGuardrailScopeTest.cs new file mode 100644 index 00000000..f429369e --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/ApplyGuardrailScopeTest.cs @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.Observability.Tests.Tracing.Scopes; + +using System; +using System.Linq; +using FluentAssertions; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; + +[TestClass] +public sealed class ApplyGuardrailScopeTest : ActivityTest +{ + [TestMethod] + public void Start_SetsRequiredAttributes() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiOperationNameKey, OpenTelemetryConstants.ApplyGuardrailOperationName); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityDecisionTypeKey, "deny"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityTargetTypeKey, GuardrailTargetType.LlmInput); + } + + [TestMethod] + public void Start_SetsGuardianAttributes_WhenProvided() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmOutput, + decisionType: GuardrailDecisionType.Allow, + guardianName: "PII Filter", + guardianId: "guard_abc123", + guardianProviderName: "azure.ai.content_safety", + guardianVersion: "2.1.0"); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiGuardianNameKey, "PII Filter"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiGuardianIdKey, "guard_abc123"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiGuardianProviderNameKey, "azure.ai.content_safety"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiGuardianVersionKey, "2.1.0"); + } + + [TestMethod] + public void Start_SetsPolicyAttributes_WhenProvided() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.ToolCall, + decisionType: GuardrailDecisionType.Modify, + policyId: "policy_pii_v2", + policyName: "PII Protection Policy", + policyVersion: "1.0"); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityPolicyIdKey, "policy_pii_v2"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityPolicyNameKey, "PII Protection Policy"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityPolicyVersionKey, "1.0"); + } + + [TestMethod] + public void Start_SpanName_IncludesGuardianNameAndTargetType() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Allow, + guardianName: "Content Safety"); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.DisplayName.Should().Be("apply_guardrail Content Safety llm_input"); + } + + [TestMethod] + public void Start_SpanName_OmitsGuardianName_WhenNull() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmOutput, + decisionType: GuardrailDecisionType.Deny); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.DisplayName.Should().Be("apply_guardrail llm_output"); + } + + [TestMethod] + public void Start_SetsContentAttributes_WhenProvided() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmOutput, + decisionType: GuardrailDecisionType.Modify, + contentInputHash: "sha256:a3f2b8c9", + contentModified: true); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityContentInputHashKey, "sha256:a3f2b8c9"); + var modifiedTag = activity.TagObjects.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityContentModifiedKey); + modifiedTag.Value.Should().Be(true); + } + + [TestMethod] + public void RecordDecision_UpdatesDecisionType() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Allow); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordDecision(GuardrailDecisionType.Deny, "Prompt injection detected"); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityDecisionTypeKey, "deny"); + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityDecisionReasonKey, "Prompt injection detected"); + } + + [TestMethod] + public void RecordContentOutput_SetsOutputValue() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmOutput, + decisionType: GuardrailDecisionType.Modify); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordContentOutput("Redacted content here"); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityContentOutputValueKey, "Redacted content here"); + } + + [TestMethod] + public void RecordFinding_EmitsSingleEvent() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordFinding(new GuardrailFinding( + riskCategory: "prompt_injection", + riskSeverity: GuardrailRiskSeverity.High, + policyDecisionType: "deny")); + }); + + activity.Events.Should().HaveCount(1); + var evt = activity.Events.First(); + evt.Name.Should().Be(OpenTelemetryConstants.GenAiSecurityFindingEventName); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskCategoryKey).Value.Should().Be("prompt_injection"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskSeverityKey).Value.Should().Be("high"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey).Value.Should().Be("deny"); + } + + [TestMethod] + public void RecordFinding_EmitsMultipleEvents() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmOutput, + decisionType: GuardrailDecisionType.Modify); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordFinding(new GuardrailFinding( + riskCategory: "pii", + riskSeverity: GuardrailRiskSeverity.Medium, + policyDecisionType: "modify")); + scope.RecordFinding(new GuardrailFinding( + riskCategory: "toxicity", + riskSeverity: GuardrailRiskSeverity.Low)); + }); + + activity.Events.Should().HaveCount(2); + } + + [TestMethod] + public void RecordFinding_IncludesAllAttributes() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Deny); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordFinding(new GuardrailFinding( + riskCategory: "sensitive_info_disclosure", + riskSeverity: GuardrailRiskSeverity.High, + policyDecisionType: "deny", + policyId: "policy_pii_v2", + policyName: "PII Policy", + policyVersion: "2.0", + riskScore: 0.92, + riskMetadata: new[] { "pattern:ssn", "count:2" })); + }); + + var evt = activity.Events.First(); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskCategoryKey).Value.Should().Be("sensitive_info_disclosure"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskSeverityKey).Value.Should().Be("high"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityPolicyDecisionTypeKey).Value.Should().Be("deny"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityPolicyIdKey).Value.Should().Be("policy_pii_v2"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityPolicyNameKey).Value.Should().Be("PII Policy"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityPolicyVersionKey).Value.Should().Be("2.0"); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskScoreKey).Value.Should().Be(0.92); + evt.Tags.First(t => t.Key == OpenTelemetryConstants.GenAiSecurityRiskMetadataKey).Value.Should().BeEquivalentTo(new[] { "pattern:ssn", "count:2" }); + } + + [TestMethod] + public void RecordError_SetsExpectedFields() + { + const string expected = "Guardian service unavailable"; + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Allow); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + scope.RecordError(new Exception(expected)); + }); + + activity.ShouldBeError(expected); + } + + [TestMethod] + public void Start_SetsConversationId_WhenProvided() + { + var conversationId = "conv-guardrail-123"; + var details = new GuardrailDetails( + targetType: GuardrailTargetType.LlmInput, + decisionType: GuardrailDecisionType.Allow); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start( + details, + Util.GetAgentDetails(), + request: new Request(conversationId: conversationId)); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiConversationIdKey, conversationId); + } + + [TestMethod] + public void Start_SetsExternalEventId_WhenProvided() + { + var details = new GuardrailDetails( + targetType: GuardrailTargetType.Message, + decisionType: GuardrailDecisionType.Audit, + externalEventId: "evt_abc123"); + + var activity = ListenForActivity(() => + { + using var scope = ApplyGuardrailScope.Start(details, Util.GetAgentDetails()); + }); + + activity.ShouldHaveTag(OpenTelemetryConstants.GenAiSecurityExternalEventIdKey, "evt_abc123"); + } +}