Bound Cosmos diagnostics under retry storms

sdk/cosmos/.cspell.json

@@ -5,6 +5,7 @@
   "ignoreWords": [
     "accountname",
     "activityid",
+    "aggregatable",
     "ALPN",
     "apacsoutheast",
     "Appleby's",
@@ -219,6 +220,7 @@
     "pgcosmos",
     "PIDS",
     "Pigw",
+    "pingpong",
     "pinning",
     "Pkrange",
     "pkranges",
@@ -255,6 +257,7 @@
     "rhost",
     "RNTBD",
     "rgby",
+    "rollup",
     "roundtripped",
     "roundtrips",
     "RTLD",
@@ -305,6 +308,7 @@
     "ukwest",
     "unavail",
     "uncollapsed",
+    "uncompacted",
     "uncontended",
     "undrained",
     "unfaulted",
@@ -337,6 +341,10 @@
     "usseceast",
     "ussecwest",
     "ussecwestcentral",
+    "varint",
+    "varints",
+    "Vecs",
+    "verbosities",
     "vnext",
     "westcentralus",
     "westeurope",

sdk/cosmos/azure_data_cosmos/tests/framework/test_client.rs

@@ -101,10 +101,10 @@ pub fn assert_region_not_contacted(
         .count();
     assert_eq!(
         on_region, 0,
-        "expected zero tracked requests on excluded region {:?}, but {} of {} requests landed there (regions contacted: {:?})",
+        "expected zero tracked requests on excluded region {:?}, but {} of {} retained requests landed there (regions contacted: {:?})",
         excluded_region,
         on_region,
-        diagnostics.request_count(),
+        requests.len(),
         diagnostics.regions_contacted()
     );
 }

sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md

@@ -29,6 +29,11 @@
 
 - The `User-Agent` header now advertises enabled client features (PPCB, HTTP/2) via the cross-SDK `|F<HEX>` feature-flag token, consistent with the .NET and Java Cosmos SDKs. ([#4635](https://github.com/Azure/azure-sdk-for-rust/pull/4635))
 - Added support for using a native query planning library to generate query plans locally, avoiding a Gateway round-trip on cross-partition queries. Gated behind the `__internal_native_query_plan` feature flag. ([#4554](https://github.com/Azure/azure-sdk-for-rust/pull/4554))
+- Added a prototype **diagnostics capture engine** (`diagnostics::capture`) behind the **off-by-default `capture_engine` feature**. It is **not** the driver's default diagnostics path: the always-on `DiagnosticsContextBuilder` continues to produce the diagnostics surfaced by `CosmosResponse::diagnostics()` with full fidelity. What ships unconditionally from this module is the **gate** — a `DiagnosticsPolicy` (`Threshold`/`Always`, default `Always`) plus `should_build`, evaluated at operation end to decide whether the builder-produced context is *also* exposed through the new gated accessors `CosmosResponse::capture_diagnostics()` and `CosmosError::capture_diagnostics()`. The gate never builds the surfaced context; it only governs exposure. There is no `Off` mode — diagnostics are always collected so that every operation stays diagnosable. Behind `capture_engine`, the engine explores a future capture-then-reconstruct path: a lock-free, pooled hot-path recorder appends to a typed, two-list **event log** (an `EventLogStorage`: `Vec<Span>` + `Vec<Attr>`) — recording an event is a single `Vec::push`. A `Span` carries a `kind`, an `Option<SpanId>` parent (`SpanId` is a `NonZero` index+1, so `Option<SpanId>` uses the niche), and op-relative `TimeOffset` start/end; an `Attr` is a typed key/value tagged with its `SpanId`, with `AttrValue` keeping the path allocation-light (`U64`/`F64`/first-class `Status(CosmosStatus)` plus `StaticStr(&'static str)`, `SharedStr(Arc<str>)`, owned `Str(Box<str>)`). The pooled storage is leased through an `EventLog` RAII handle that returns it to the pool on drop. Past the gate the flat lists are replayed onto a `DiagnosticsContextBuilder`; the reconstruction carries the captured per-attempt facets (pipeline type, transport security/kind, HTTP version) and the server-reported duration, reconstructed from what was recorded rather than hardcoded, and a parity harness validates it against the builder. The path stays behind the feature because the live driver pipeline does not yet feed the recorder. The public `DiagnosticsContext` type, the non-optional `response.diagnostics()` accessor, and the `azure_data_cosmos` boundary are **unchanged** (additive, non-breaking). See `DIAGNOSTICS-CAPTURE.md`.
+- Added a `.NET CosmosDiagnostics`-style top-level **`summary`** block to `DiagnosticsContext` (`DiagnosticsSummary`, reachable via `DiagnosticsContext::summary()`). It aggregates over the collected requests — `(status, sub-status)` histogram, total request charge, request/retry/throttle counts, regions contacted, final status, top error, total elapsed, and the client **`user_agent`** (SDK + runtime identity, via `DiagnosticsSummary::user_agent()`) — and is **computed lazily on first access** (never on the hot path) and cached via `OnceLock`, so a context whose diagnostics are never inspected pays nothing for it. Serialized as the first section of the diagnostics output; the `user_agent` is omitted when the driver did not supply one, so existing output is unchanged.
+- Added a **`DiagnosticsEncoding`** client option on `DriverOptions` (`DriverOptionsBuilder::with_diagnostics_encoding` / `DriverOptions::diagnostics_encoding`) controlling how diagnostics render to a string via the new `DiagnosticsContext::encode(encoding)` and `CosmosResponse::diagnostics_string(encoding)`: `Json` (pretty, **default** — output unchanged), `Compact` (minified JSON), and `Encoded` (base64 of the compact JSON, decodable, for size-sensitive logging). Additive and non-breaking.
+- Added serializable **wall-clock timestamps** (RFC 3339 / ISO 8601) to the diagnostics output. `DiagnosticsContext` (and its `summary`) now carry an operation `start_time`/`end_time`, and each `RequestDiagnostics` attempt carries a `start_time` plus an optional `end_time`, captured with `azure_core::time::OffsetDateTime` alongside the existing `Instant`-based durations. These absolute timestamps let diagnostics be correlated against server-side and external logs. Additive and non-breaking: the fields exist only when a context is built, durations are unchanged, and an in-flight attempt's `end_time` is omitted (`skip_serializing_if`).
+- Added **retry-storm compaction** to the always-on `DiagnosticsContext`, giving a bounded-size guarantee under 4xx/5xx retry storms. When an operation accumulates more per-attempt records than the new configurable `DiagnosticsOptions::max_request_diagnostics` cap (`DiagnosticsOptionsBuilder::with_max_request_diagnostics`, env `AZURE_COSMOS_DIAGNOSTICS_MAX_REQUESTS`, default `512`, min `16`), runs of near-identical consecutive retries (same region / endpoint / status / sub-status / execution-context) are collapsed at operation finalization into first + last + a count, with an order-robust global key-bucket fallback that keeps the retained count within the cap even under a region ping-pong. The aggregate `summary` is computed from the full attempt list *before* compaction, so the `(status, sub-status)` histogram, retry/throttle counts, total RU and regions stay exact; `DiagnosticsContext::request_count()` still reports the true total, and new accessors `DiagnosticsContext::retained_request_count()` and `DiagnosticsContext::compaction()` (returning the new `CompactionInfo` / `CompactedRun`) expose the bounded retained count and a per-run rollup. The per-run rollup and the retained list are themselves bounded so the serialized artifact stays independent of storm *cardinality* (e.g. a `410` fan-out across thousands of physical-partition endpoints): `CompactionInfo` reports `total_runs`, and `omitted_runs` / `omitted_request_count` when the rollup is capped, plus `retained_truncated` when the retained list hit the cap — so every drop is **explicit**, never silent. Under the global key-bucket fallback the retained records and the per-run rollup are drawn from the **same** cap-bounded set of largest runs, so every retained attempt has a matching run in the rollup (coherent for a span emitter). The bounded-size guarantee applies to the finalized serialized artifact and retained list, **not** to live mid-operation memory (the in-flight per-attempt list still grows one entry per attempt; only finalization bounds it). Detailed JSON gains a top-level `compaction` object only when compaction fires (`skip_serializing_if`), so output stays byte-identical for normal operations. Additive and non-breaking.
 - Restructured the client / runtime options layering on the driver. Two new nested option groups, a per-client overrides surface on `DriverOptionsBuilder`, and a single canonical `AZURE_COSMOS_PPCB_*` namespace for partition-failover environment variables. The driver now consumes partition-failover configuration once at construction (`CosmosDriver::new` no longer fabricates an `OperationOptionsView` outside any operation context) ([#4588](https://github.com/Azure/azure-sdk-for-rust/pull/4588)):
 - Added new nested `OperationOptions::throughput_control` group (`ThroughputControlOptions` / `…Builder` / `…View`, mirroring the `ThrottlingRetryOptions` pattern). Exposes three layered fields ([#4588](https://github.com/Azure/azure-sdk-for-rust/pull/4588)):
   - `group_name: Option<ThroughputControlGroupName>` — replaces the old top-level `OperationOptions::throughput_control_group`.

sdk/cosmos/azure_data_cosmos_driver/Cargo.toml

@@ -41,6 +41,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng"] }
 
 [dev-dependencies]
 azure_identity.workspace = true
+criterion.workspace = true
 quick-xml.workspace = true
 serde = { workspace = true, features = ["derive"] }
 tokio = { workspace = true, features = [
@@ -78,6 +79,15 @@ rustls = ["reqwest", "reqwest/rustls", "__tls"]
 native_tls = ["reqwest", "reqwest/native-tls", "__tls"]
 fault_injection = []
 preview_dtx = [] # Enables preview Distributed Transaction APIs. Disabled by default and not production-ready.
+# `capture_engine` exposes the prototype deferred, append-only diagnostics
+# **capture** engine (`diagnostics::capture::{event, context, encode, pool,
+# recorder}` plus `gate::finish`). It is OFF by default and is NOT
+# production-ready: the shipping diagnostics path is the always-on
+# `DiagnosticsContextBuilder` and the unconditional Off/Threshold/Always gate.
+# The event-log engine is opt-in scaffolding for evaluating a future capture
+# -> reconstruct path (see DIAGNOSTICS-CAPTURE.md). It is not covered by SemVer
+# and may change or disappear at any time.
+capture_engine = []
 # `__internal_in_memory_emulator` exposes the in-memory query evaluator
 # (`crate::query::eval`, `crate::query::value`) used by the in-memory Cosmos DB
 # emulator. The evaluator intentionally trades full Cosmos parity for emulator
@@ -126,6 +136,11 @@ name = "multi_region_failover"
 path = "tests/multi_region_failover.rs"
 required-features = ["fault_injection"]
 
+[[test]]
+name = "live_storm_diagnostics"
+path = "tests/live_storm_diagnostics.rs"
+required-features = ["fault_injection"]
+
 [[test]]
 name = "gateway_query_plan_comparison"
 path = "tests/gateway_query_plan_comparison.rs"
@@ -135,3 +150,16 @@ required-features = ["__internal_testing"]
 name = "in_memory_emulator"
 path = "tests/in_memory_emulator.rs"
 required-features = ["__internal_in_memory_emulator"]
+
+# The capture-engine example tests exercise `diagnostics::capture::{recorder,
+# pool, finish}`, which only exist behind the off-by-default `capture_engine`
+# feature.
+[[test]]
+name = "diagnostics_examples"
+path = "tests/diagnostics_examples.rs"
+required-features = ["capture_engine"]
+
+[[bench]]
+name = "diagnostics_capture"
+harness = false
+required-features = ["capture_engine"]