diff --git a/sdk/core/azure_core/CHANGELOG.md b/sdk/core/azure_core/CHANGELOG.md index 8bd1f3fb223..672800bdb60 100644 --- a/sdk/core/azure_core/CHANGELOG.md +++ b/sdk/core/azure_core/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added `Tracer::start_span_with_options`, `Tracer::start_span_with_parent_and_options`, and `Span::end_at`, along with a `SpanOptions` struct, to allow reconstructing spans with explicit (backdated) start and end timestamps. These are additive with default implementations, so existing `Tracer`/`Span` implementations continue to work unchanged. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/core/azure_core/src/lib.rs b/sdk/core/azure_core/src/lib.rs index afe53ce4f33..334518975be 100644 --- a/sdk/core/azure_core/src/lib.rs +++ b/sdk/core/azure_core/src/lib.rs @@ -29,8 +29,8 @@ pub mod tracing { pub use crate::http::policies::PublicApiInstrumentationInformation; pub use azure_core_macros::{client, function, new, subclient}; pub use typespec_client_core::tracing::{ - AsAny, Attribute, AttributeArray, AttributeValue, Span, SpanGuard, SpanKind, SpanStatus, - Tracer, TracerProvider, + AsAny, Attribute, AttributeArray, AttributeValue, Span, SpanGuard, SpanKind, SpanOptions, + SpanStatus, Tracer, TracerProvider, }; } diff --git a/sdk/core/azure_core_opentelemetry/CHANGELOG.md b/sdk/core/azure_core_opentelemetry/CHANGELOG.md index 072b97827c0..dcc7e7bd154 100644 --- a/sdk/core/azure_core_opentelemetry/CHANGELOG.md +++ b/sdk/core/azure_core_opentelemetry/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Implemented span backdating (`Tracer::start_span_with_options`, `Tracer::start_span_with_parent_and_options`, and `Span::end_at`) by mapping to the OpenTelemetry `SpanBuilder::with_start_time` and `Span::end_with_timestamp` APIs, so late-bound (tail-sampled) spans can be emitted with their original timestamps. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/core/azure_core_opentelemetry/src/span.rs b/sdk/core/azure_core_opentelemetry/src/span.rs index 136814037d3..b85d83b8a76 100644 --- a/sdk/core/azure_core_opentelemetry/src/span.rs +++ b/sdk/core/azure_core_opentelemetry/src/span.rs @@ -6,6 +6,7 @@ use crate::attributes::AttributeValue as ConversionAttributeValue; use azure_core::{ http::headers::{HeaderName, HeaderValue}, + time::OffsetDateTime, tracing::{AsAny, AttributeValue, Span, SpanGuard, SpanStatus}, }; use opentelemetry::{ @@ -13,7 +14,7 @@ use opentelemetry::{ trace::TraceContextExt, }; use opentelemetry_sdk::propagation::TraceContextPropagator; -use std::{error::Error as StdError, sync::Arc}; +use std::{error::Error as StdError, sync::Arc, time::SystemTime}; /// newtype for Azure Core SpanKind to enable conversion to OpenTelemetry SpanKind pub(crate) struct OpenTelemetrySpanKind(pub azure_core::tracing::SpanKind); @@ -64,6 +65,12 @@ impl Span for OpenTelemetrySpan { self.context.span().end(); } + fn end_at(&self, end_time: OffsetDateTime) { + self.context + .span() + .end_with_timestamp(SystemTime::from(end_time)); + } + fn span_id(&self) -> [u8; 8] { self.context.span().span_context().span_id().to_bytes() } @@ -150,12 +157,16 @@ impl Drop for OpenTelemetrySpanGuard { mod tests { use crate::telemetry::OpenTelemetryTracerProvider; use azure_core::http::{Context as AzureContext, Url}; - use azure_core::tracing::{Attribute, AttributeValue, SpanKind, SpanStatus, TracerProvider}; + use azure_core::time::{Duration, OffsetDateTime}; + use azure_core::tracing::{ + Attribute, AttributeValue, SpanKind, SpanOptions, SpanStatus, TracerProvider, + }; use opentelemetry::trace::TraceContextExt; use opentelemetry::{Context, Key, KeyValue, Value}; use opentelemetry_sdk::trace::{in_memory_exporter::InMemorySpanExporter, SdkTracerProvider}; use std::io::{Error, ErrorKind}; use std::sync::Arc; + use std::time::SystemTime; use tracing::trace; fn create_exportable_tracer_provider() -> (Arc, InMemorySpanExporter) { @@ -304,6 +315,92 @@ mod tests { } } + #[test] + fn test_open_telemetry_span_backdated() { + let (otel_tracer_provider, otel_exporter) = create_exportable_tracer_provider(); + let tracer_provider = OpenTelemetryTracerProvider::new(otel_tracer_provider); + let tracer = tracer_provider.get_tracer(Some("Backdated"), "test", Some("0.1.0")); + + // Choose timestamps clearly in the past so we can prove the span is *not* + // stamped at "now". + let now = OffsetDateTime::now_utc(); + let past_start = now - Duration::seconds(3600); + let past_end = now - Duration::seconds(1800); + + let span = tracer.start_span_with_options( + "backdated_span".into(), + SpanKind::Client, + vec![], + SpanOptions { + start_time: Some(past_start), + }, + ); + span.end_at(past_end); + + let spans = otel_exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + let span = &spans[0]; + assert_eq!(span.name, "backdated_span"); + // The injected past timestamps must be preserved exactly, not replaced with "now". + assert_eq!(span.start_time, SystemTime::from(past_start)); + assert_eq!(span.end_time, SystemTime::from(past_end)); + assert!(span.start_time < span.end_time); + assert!(span.end_time < SystemTime::from(now)); + } + + #[test] + fn test_open_telemetry_span_backdated_with_parent() { + let (otel_tracer_provider, otel_exporter) = create_exportable_tracer_provider(); + let tracer_provider = OpenTelemetryTracerProvider::new(otel_tracer_provider); + let tracer = tracer_provider.get_tracer(Some("Backdated"), "test", Some("0.1.0")); + + // Reconstruct a completed operation: a backdated root with a backdated attempt child. + let now = OffsetDateTime::now_utc(); + let root_start = now - Duration::seconds(600); + let child_start = now - Duration::seconds(590); + let child_end = now - Duration::seconds(585); + let root_end = now - Duration::seconds(580); + + let root = tracer.start_span_with_options( + "operation".into(), + SpanKind::Client, + vec![], + SpanOptions { + start_time: Some(root_start), + }, + ); + let child = tracer.start_span_with_parent_and_options( + "attempt".into(), + SpanKind::Client, + vec![], + root.clone(), + SpanOptions { + start_time: Some(child_start), + }, + ); + child.end_at(child_end); + root.end_at(root_end); + + let spans = otel_exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 2); + + let root_span = spans.iter().find(|s| s.name == "operation").unwrap(); + let child_span = spans.iter().find(|s| s.name == "attempt").unwrap(); + + assert_eq!(root_span.start_time, SystemTime::from(root_start)); + assert_eq!(root_span.end_time, SystemTime::from(root_end)); + assert_eq!(child_span.start_time, SystemTime::from(child_start)); + assert_eq!(child_span.end_time, SystemTime::from(child_end)); + assert!(root_span.end_time < SystemTime::from(now)); + + // The child must be parented to the backdated root. + assert_ne!( + child_span.parent_span_id, + opentelemetry::trace::SpanId::INVALID + ); + assert_eq!(child_span.parent_span_id.to_bytes(), root.span_id()); + } + #[test] fn test_open_telemetry_span_set_attribute() { let (otel_tracer_provider, otel_exporter) = create_exportable_tracer_provider(); diff --git a/sdk/core/azure_core_opentelemetry/src/tracer.rs b/sdk/core/azure_core_opentelemetry/src/tracer.rs index 5d123d25909..c8f83c1fb50 100644 --- a/sdk/core/azure_core_opentelemetry/src/tracer.rs +++ b/sdk/core/azure_core_opentelemetry/src/tracer.rs @@ -6,13 +6,14 @@ use crate::{ span::{OpenTelemetrySpan, OpenTelemetrySpanKind}, }; -use azure_core::tracing::{SpanKind, Tracer}; +use azure_core::time::OffsetDateTime; +use azure_core::tracing::{SpanKind, SpanOptions, Tracer}; use opentelemetry::{ global::BoxedTracer, trace::{TraceContextExt, Tracer as OpenTelemetryTracerTrait}, Context, KeyValue, }; -use std::{borrow::Cow, fmt::Debug, sync::Arc}; +use std::{borrow::Cow, fmt::Debug, sync::Arc, time::SystemTime}; pub struct OpenTelemetryTracer { namespace: Option<&'static str>, @@ -27,6 +28,45 @@ impl OpenTelemetryTracer { inner: tracer, } } + + /// Builds a span within `context`, optionally backdated to `start_time`. + fn build_span( + &self, + name: Cow<'static, str>, + kind: SpanKind, + attributes: Vec, + start_time: Option, + context: Context, + ) -> Arc { + let mut span_builder = opentelemetry::trace::SpanBuilder::from_name(name) + .with_kind(OpenTelemetrySpanKind(kind).into()) + .with_attributes( + attributes + .iter() + .map(|attr| KeyValue::from(OpenTelemetryAttribute(attr.clone()))), + ); + if let Some(start_time) = start_time { + span_builder = span_builder.with_start_time(SystemTime::from(start_time)); + } + let span = self.inner.build_with_context(span_builder, &context); + + OpenTelemetrySpan::new(context.with_span(span)) + } + + /// Extracts the OpenTelemetry context from a parent span. + fn parent_context(parent: &Arc) -> Context { + parent + .as_any() + .downcast_ref::() + .unwrap_or_else(|| { + panic!( + "Could not downcast parent span to OpenTelemetrySpan. Actual type: {}", + std::any::type_name::() + ) + }) + .context() + .clone() + } } impl Debug for OpenTelemetryTracer { @@ -48,17 +88,23 @@ impl Tracer for OpenTelemetryTracer { kind: SpanKind, attributes: Vec, ) -> Arc { - let span_builder = opentelemetry::trace::SpanBuilder::from_name(name) - .with_kind(OpenTelemetrySpanKind(kind).into()) - .with_attributes( - attributes - .iter() - .map(|attr| KeyValue::from(OpenTelemetryAttribute(attr.clone()))), - ); - let context = Context::current(); - let span = self.inner.build_with_context(span_builder, &context); + self.build_span(name, kind, attributes, None, Context::current()) + } - OpenTelemetrySpan::new(context.with_span(span)) + fn start_span_with_options( + &self, + name: Cow<'static, str>, + kind: SpanKind, + attributes: Vec, + options: SpanOptions, + ) -> Arc { + self.build_span( + name, + kind, + attributes, + options.start_time, + Context::current(), + ) } fn start_span_with_parent( @@ -68,29 +114,20 @@ impl Tracer for OpenTelemetryTracer { attributes: Vec, parent: Arc, ) -> Arc { - let span_builder = opentelemetry::trace::SpanBuilder::from_name(name) - .with_kind(OpenTelemetrySpanKind(kind).into()) - .with_attributes( - attributes - .iter() - .map(|attr| KeyValue::from(OpenTelemetryAttribute(attr.clone()))), - ); - - // Cast the parent span to Any type - let context = parent - .as_any() - .downcast_ref::() - .unwrap_or_else(|| { - panic!( - "Could not downcast parent span to OpenTelemetrySpan. Actual type: {}", - std::any::type_name::() - ) - }) - .context() - .clone(); - let span = self.inner.build_with_context(span_builder, &context); + let context = Self::parent_context(&parent); + self.build_span(name, kind, attributes, None, context) + } - OpenTelemetrySpan::new(context.with_span(span)) + fn start_span_with_parent_and_options( + &self, + name: Cow<'static, str>, + kind: SpanKind, + attributes: Vec, + parent: Arc, + options: SpanOptions, + ) -> Arc { + let context = Self::parent_context(&parent); + self.build_span(name, kind, attributes, options.start_time, context) } } diff --git a/sdk/core/typespec_client_core/CHANGELOG.md b/sdk/core/typespec_client_core/CHANGELOG.md index df53893335e..d486e3a34cc 100644 --- a/sdk/core/typespec_client_core/CHANGELOG.md +++ b/sdk/core/typespec_client_core/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added `Tracer::start_span_with_options`, `Tracer::start_span_with_parent_and_options`, and `Span::end_at`, along with a `SpanOptions` struct, to allow reconstructing spans with explicit (backdated) start and end timestamps. These are additive with default implementations, so existing `Tracer`/`Span` implementations continue to work unchanged. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/core/typespec_client_core/src/tracing/mod.rs b/sdk/core/typespec_client_core/src/tracing/mod.rs index 42341da43de..20d8552a67a 100644 --- a/sdk/core/typespec_client_core/src/tracing/mod.rs +++ b/sdk/core/typespec_client_core/src/tracing/mod.rs @@ -4,6 +4,7 @@ //! Distributed tracing trait definitions //! use crate::http::{Context, Request}; +use crate::time::OffsetDateTime; use std::{borrow::Cow, fmt::Debug, sync::Arc}; /// Overall architecture for distributed tracing in the SDK. @@ -39,6 +40,33 @@ pub trait TracerProvider: Send + Sync + Debug { ) -> Arc; } +/// Options that customize how a span is started. +/// +/// `SpanOptions` is passed to [`Tracer::start_span_with_options`] and +/// [`Tracer::start_span_with_parent_and_options`]. It implements [`Default`], so new options +/// can be added in the future without changing the [`Tracer`] method signatures. Construct it +/// with [`Default::default`] and struct update syntax: +/// +/// ``` +/// use typespec_client_core::tracing::SpanOptions; +/// use typespec_client_core::time::OffsetDateTime; +/// +/// let options = SpanOptions { +/// start_time: Some(OffsetDateTime::now_utc()), +/// ..Default::default() +/// }; +/// ``` +#[derive(Debug, Clone, Default)] +pub struct SpanOptions { + /// An explicit start time to record for the span. + /// + /// When `None`, the span starts at the current time. When `Some`, the span is + /// *backdated* to the given time, letting a caller reconstruct a span for an operation + /// that has *already completed* — required for tail-based (late-bound) sampling, where + /// the decision to emit a span is made after the operation finishes. + pub start_time: Option, +} + /// The `Tracer` trait is responsible for creating spans and managing the active span in distributed tracing. /// /// This trait defines methods for starting new spans, starting spans with a parent, and retrieving the namespace of the tracer. @@ -83,6 +111,74 @@ pub trait Tracer: Send + Sync + Debug { parent: Arc, ) -> Arc; + /// Starts a new span with the given name and type, customized by [`SpanOptions`]. + /// + /// The newly created span will have the "current" span as a parent. [`SpanOptions`] lets + /// the caller, for example, *backdate* the span via [`SpanOptions::start_time`] so it can + /// reconstruct a span for an operation that has *already completed*. This is required for + /// tail-based (late-bound) sampling, where the decision to emit a span is made after the + /// operation finishes. + /// + /// # Arguments + /// - `name`: The name of the span to start. + /// - `kind`: The type of the span to start. + /// - `attributes`: A vector of attributes to associate with the span. + /// - `options`: Additional options, such as an explicit start time, for the span. + /// + /// # Returns + /// An `Arc` representing the started span. + /// + /// # Note + /// The default implementation ignores `options` and delegates to [`Tracer::start_span`] + /// so that existing implementations remain source-compatible. Implementations backed by a + /// tracing system that supports these options (such as the OpenTelemetry bridge) override + /// this to honor them. + fn start_span_with_options( + &self, + name: Cow<'static, str>, + kind: SpanKind, + attributes: Vec, + options: SpanOptions, + ) -> Arc { + let _ = options; + self.start_span(name, kind, attributes) + } + + /// Starts a new child span with the given name, type, and parent span, customized by [`SpanOptions`]. + /// + /// This is the [`SpanOptions`] variant of [`Tracer::start_span_with_parent`]. It lets a + /// caller reconstruct a child span (for example, a single retry attempt) under an explicit + /// parent, optionally *backdated* via [`SpanOptions::start_time`]. + /// + /// # Arguments + /// - `name`: The name of the span to start. + /// - `kind`: The type of the span to start. + /// - `attributes`: A vector of attributes to associate with the span. + /// - `parent`: The parent span to use for the new span. + /// - `options`: Additional options, such as an explicit start time, for the span. + /// + /// # Returns + /// An `Arc` representing the started span. + /// + /// # Note + /// The default implementation ignores `options` and delegates to + /// [`Tracer::start_span_with_parent`] so that existing implementations remain + /// source-compatible. Implementations backed by a tracing system that supports these + /// options (such as the OpenTelemetry bridge) override this to honor them. + /// + /// Note: This method may panic if the parent span cannot be downcasted to the expected type. + fn start_span_with_parent_and_options( + &self, + name: Cow<'static, str>, + kind: SpanKind, + attributes: Vec, + parent: Arc, + options: SpanOptions, + ) -> Arc { + let _ = options; + self.start_span_with_parent(name, kind, attributes, parent) + } + /// Returns the namespace the tracer was configured with (if any). /// /// # Returns @@ -149,6 +245,26 @@ pub trait Span: AsAny + Send + Sync { /// Ends the current span. fn end(&self); + /// Ends the current span at an explicit end time. + /// + /// This is the backdating variant of [`Span::end`]. It lets a caller close a + /// reconstructed span at the timestamp the operation actually finished, rather than + /// "now" — the counterpart to [`SpanOptions::start_time`] for late-bound (tail-sampled) + /// spans. + /// + /// # Arguments + /// - `end_time`: The explicit end time to record for the span. + /// + /// # Note + /// The default implementation ignores `end_time` and delegates to [`Span::end`] so that + /// existing implementations remain source-compatible. Implementations backed by a + /// tracing system that supports explicit timestamps (such as the OpenTelemetry bridge) + /// override this to honor `end_time`. + fn end_at(&self, end_time: OffsetDateTime) { + let _ = end_time; + self.end(); + } + /// Sets the status of the current span. /// # Arguments /// - `status`: The status to set for the current span.