From 004ba70795361547a820e6dff729913d5a24be9b Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Tue, 7 Jul 2026 14:12:14 -0700 Subject: [PATCH 1/4] Hedge the Collection Read metadata call Extends cross-region read hedging to the Collection Read (container properties) metadata call by adding ResourceType::DocumentCollection to HEDGEABLE_RESOURCE_TYPES. This metadata read is on the critical path of nearly every operation (cold-start cache population and steady-state refresh), so hedging it trims the long tail when the primary region is briefly slow. Scoping is precise. Because HEDGEABLE_OPERATION_TYPES stays [Read], the only newly-eligible tuple is (DocumentCollection, Read) = the container point read. Container writes (Create/Delete) are excluded by the is_read_only() guard, and the container feed reads (read_all_containers -> ReadFeed, query_containers -> Query) are excluded because their operation types are not hedgeable. Crucially, user change-feed / cross-partition feed reads (Document + ReadFeed) remain unhedged -- widening operation types to ReadFeed (needed for the PartitionKeyRange routing-map read) would have caught those too, so that read is deliberately left to a follow-up that adds (resource, operation) tuple gating plus first-page-only continuation pinning. Adds should_hedge tests covering: container read eligible (multi-region) and ineligible (single region); read_all_containers, query_containers, and a user change-feed read all NOT hedged (the over-broadening guard). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../driver/pipeline/hedging_eligibility.rs | 118 ++++++++++++++++-- 1 file changed, 109 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs index 9fab10e0f7e..be1aa43b0e7 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs @@ -30,16 +30,40 @@ use crate::{ /// this constant is the upper bound. const DEFAULT_THRESHOLD_CAP: Duration = Duration::from_millis(1000); -/// Resource types eligible for cross-region hedging in the current phase. +/// Resource types eligible for cross-region hedging. /// -/// Subsequent phases widen this single constant — no other change to -/// [`should_hedge`] is required. -const HEDGEABLE_RESOURCE_TYPES: &[ResourceType] = &[ResourceType::Document]; - -/// Operation types eligible for cross-region hedging in the current phase. +/// - [`ResourceType::Document`] — data-plane point reads. +/// - [`ResourceType::DocumentCollection`] — the Collection Read that resolves +/// container properties. This metadata read sits on the critical path of +/// nearly every operation (cold-start cache population and steady-state +/// refresh), so hedging it trims the long tail when the primary region is +/// briefly slow. +/// +/// This pairs with [`HEDGEABLE_OPERATION_TYPES`] = `[Read]`: the only eligible +/// `(resource, operation)` tuples are therefore `(Document, Read)` and +/// `(DocumentCollection, Read)`. Container **writes** (`Create` / `Delete`) are +/// excluded by the `is_read_only()` guard, and the container **feed** reads +/// (`read_all_containers` → `ReadFeed`, `query_containers` → `Query`) are +/// excluded because their operation types are not in [`HEDGEABLE_OPERATION_TYPES`]. +/// +/// The other critical-path metadata read — the PartitionKeyRange `ReadFeed` +/// that builds the routing map — is intentionally **not** hedged here: making +/// it eligible would require adding `ReadFeed` to [`HEDGEABLE_OPERATION_TYPES`], +/// which (because `Document` is already hedgeable) would also make user +/// change-feed / cross-partition feed reads (`Document` + `ReadFeed`) eligible. +/// Scoping that correctly needs `(resource, operation)` tuple gating plus a +/// first-page-only / winning-region continuation-pinning story, and is left to +/// a follow-up. +const HEDGEABLE_RESOURCE_TYPES: &[ResourceType] = + &[ResourceType::Document, ResourceType::DocumentCollection]; + +/// Operation types eligible for cross-region hedging. /// -/// Future phases will append feed-style operations -/// (`Query` / `ReadFeed` / `QueryPlan`) and metadata reads. +/// Restricted to `Read` (point reads) so that only the point-read metadata and +/// data-plane paths hedge. Feed-style operations (`ReadFeed` / `Query` / +/// `QueryPlan`) are handled in a future phase — see the note on +/// [`HEDGEABLE_RESOURCE_TYPES`] about why widening this to `ReadFeed` requires +/// tuple gating to avoid unintentionally hedging user feed reads. const HEDGEABLE_OPERATION_TYPES: &[OperationType] = &[OperationType::Read]; /// Returns `true` when the operation is eligible for cross-region hedging. @@ -356,6 +380,39 @@ mod tests { CosmosOperation::read_database(db) } + fn db_reference() -> DatabaseReference { + let account = AccountReference::with_master_key( + Url::parse("https://acct.documents.azure.com/").unwrap(), + "k", + ); + DatabaseReference::from_name(account, "db") + } + + /// Collection Read (`DocumentCollection` + `Read`) — the metadata read this + /// phase makes hedgeable. + fn read_container_operation() -> CosmosOperation { + CosmosOperation::read_container_by_name(db_reference(), "c") + } + + /// `read_all_containers` (`DocumentCollection` + `ReadFeed`) — a container + /// *feed* read that must NOT be hedged. + fn read_all_containers_operation() -> CosmosOperation { + CosmosOperation::read_all_containers(db_reference()) + } + + /// `query_containers` (`DocumentCollection` + `Query`) — must NOT be hedged. + fn query_containers_operation() -> CosmosOperation { + CosmosOperation::query_containers(db_reference()) + } + + /// User change-feed read (`Document` + `ReadFeed`) — must NOT be hedged. + /// This guards against the over-broadening trap: widening resource types to + /// include `DocumentCollection` must not make any `ReadFeed` eligible. + fn read_all_items_operation() -> CosmosOperation { + let container = fake_container_reference(); + CosmosOperation::read_all_items(container, PartitionKey::from("pk")) + } + fn enabled_strategy() -> HedgingStrategy { HedgingStrategy::new(HedgeThreshold::new(Duration::from_millis(500)).unwrap()) } @@ -413,12 +470,55 @@ mod tests { #[test] fn should_hedge_non_document() { - // Reads against non-Document resource types are excluded in Phase 1. + // Reads against non-hedgeable resource types (e.g. Database) are excluded. let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); let op = read_database_operation(); assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); } + #[test] + fn should_hedge_container_read_multi_region() { + // Collection Read (DocumentCollection + Read) is hedgeable: it is a + // point read of container properties on the metadata critical path. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_container_operation(); + assert!(should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_read_single_region() { + let state = account_state_with_regions(&[Region::EAST_US]); + let op = read_container_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_feed_not_hedged() { + // read_all_containers is DocumentCollection + ReadFeed — a feed read, + // NOT eligible (ReadFeed is not in HEDGEABLE_OPERATION_TYPES). + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_all_containers_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_query_not_hedged() { + // query_containers is DocumentCollection + Query — NOT eligible. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = query_containers_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_document_readfeed_not_hedged() { + // Over-broadening guard: a user change-feed read (Document + ReadFeed) + // must remain non-hedgeable even though Document is a hedgeable resource + // type — because ReadFeed is not in HEDGEABLE_OPERATION_TYPES. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_all_items_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + #[test] fn should_hedge_disabled_override() { // `None` represents Disabled at any layer — short-circuits before From 89d2491c6113ac08422f0d00fe928025a91ffb0a Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Tue, 7 Jul 2026 14:13:30 -0700 Subject: [PATCH 2/4] Add CHANGELOG entries for Collection Read hedging Document the extension of cross-region read hedging to the Collection Read metadata call in both the driver (azure_data_cosmos_driver) and public (azure_data_cosmos) changelogs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/azure_data_cosmos/CHANGELOG.md | 1 + sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md index 1f6baeb657f..56f9a2f2f09 100644 --- a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - Added `TlsBackend` (re-exported) and a `tls_backend` option on `ConnectionPoolOptions` (`ConnectionPoolOptionsBuilder::with_tls_backend`), defaulting to `TlsBackend::Rustls`, available under the `rustls` feature, to pin the TLS backend used by the transport. This is additive and changes no behavior for the default (rustls) build; it only has an effect in builds that compile in multiple reqwest TLS backends, where reqwest would otherwise default to native-tls and the driver now pins rustls instead. ([#4649](https://github.com/Azure/azure-sdk-for-rust/pull/4649)) +- Cross-region read hedging now also covers the Collection Read metadata call (resolving container properties), in addition to data-plane point reads. When hedging is enabled (the default for multi-region accounts), the container-properties read is speculatively dispatched to a second preferred region after the threshold, reducing tail latency on the first operation against a container and on cache refreshes. No API change; enable or disable it via the existing `AvailabilityStrategy` / `AZURE_COSMOS_HEDGING_ENABLED` controls. ([#4710](https://github.com/Azure/azure-sdk-for-rust/pull/4710)) ### Breaking Changes diff --git a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md index 7c342ffd199..e05bcd75a7c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - Added `TlsBackend` (currently `TlsBackend::Rustls`, the default) and a `tls_backend` option on `ConnectionPoolOptions` (`ConnectionPoolOptionsBuilder::with_tls_backend` / `ConnectionPoolOptions::tls_backend`), available under the `rustls` feature. The driver asserts the selected backend on the `reqwest` transport, giving a supported way to pin the TLS backend without direct transport access. This is additive and changes no behavior for the default (rustls-only) build, where reqwest already negotiates rustls; it only has an effect in builds that compile in multiple reqwest TLS backends (e.g. `rustls` plus `native_tls`, absent reqwest's `http3` feature), where reqwest would otherwise default to native-tls and the driver now pins rustls instead. ([#4649](https://github.com/Azure/azure-sdk-for-rust/pull/4649)) +- Extended cross-region read hedging to the Collection Read metadata call (container properties). When hedging is enabled (the default for multi-region accounts), the driver now speculatively dispatches the container-properties read to a second preferred region after the threshold elapses, trimming the long tail on cold-start cache population and steady-state refresh. Scoped precisely to `(DocumentCollection, Read)`: container feed reads (`read_all_containers`), queries, and writes stay unhedged, and user change-feed / cross-partition feed reads (`Document` + `ReadFeed`) are unaffected. The PartitionKeyRange routing-map `ReadFeed` remains a separate future phase. ([#4710](https://github.com/Azure/azure-sdk-for-rust/pull/4710)) ### Breaking Changes From 56f47d9473b2ab803b6e4cd7555887761bff1770 Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Tue, 7 Jul 2026 14:52:25 -0700 Subject: [PATCH 3/4] Sync HEDGING_SPEC + add e2e Collection Read test Implements review feedback on the Collection Read hedging widen. HEDGING_SPEC.md was still describing Phase 1 as {Document}-only, so the doc had drifted from the code now that (DocumentCollection, Read) is hedgeable. Updated the section 1 operation-type scope table, the section 5.1 phase-allowed ResourceType table, the section 15.1 unit-test table (corrected should_hedge_non_document and listed the five new tests), and the section 16 Phase 1/Phase 2 sections. Also resolved the spec's open "metadata cache invalidation" question for the Container point read: fastest-wins is safe because the driver only consumes the immutable _rid and partition-key definition from that payload. Added hedging_container_read_primary_slow, the metadata analogue of hedging_read_primary_slow, which injects an 800ms delay on the East US MetadataReadContainer and asserts the alternate region wins. This is end-to-end proof through the pipeline that the HEDGEABLE_RESOURCE_TYPES widen actually hedges the container-properties read, not just that should_hedge returns true at the unit level. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../docs/HEDGING_SPEC.md | 65 +++++++---- .../tests/in_memory_emulator_tests/hedging.rs | 102 +++++++++++++++++- 2 files changed, 145 insertions(+), 22 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md index 597ad4bda82..751cd01eacc 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md @@ -94,7 +94,8 @@ design review. | Queries (`QueryItems`) — page-level | ❌ | ✅ | ✅ | | `ReadMany` — page-level | ❌ | ✅ | ✅ | | Change feed — page-level | ❌ | ✅ | ✅ | -| Metadata operations (Database / Container / Offer / Throughput) | ❌ | ✅ | ✅ | +| Container Read (properties) — point read (`DocumentCollection` + `Read`) | ✅ | ✅ | ✅ | +| Other metadata reads (Database / Offer / Throughput / PartitionKeyRange) | ❌ | ✅ | ✅ | | Document writes (Create/Replace/Upsert/Delete/Patch) — any topology | ❌ | ❌ | ❌ | | Stored procedure execution (`ExecuteJavaScript`) | ❌ | ❌ | 🟡 candidate | @@ -104,13 +105,18 @@ design review. > procedure execution is a standalone server-side execution and > deferred to Future. -Phase 1 ships document point reads only — the smallest correct surface -that exercises `execute_hedged()`, region pinning, cancellation, and the -PPCB feedback loop end-to-end. Phase 2 widens to feed-style operations -(Query / ReadMany / ChangeFeed), each hedged **per page**, plus -metadata operations. The exact integration with the `FeedRange` -abstraction is being co-designed with the feed-operation spec (see -§16). Writes are not in scope for any phase. +Phase 1 ships document point reads plus the Collection Read metadata +call (`DocumentCollection` + `Read`, i.e. resolving container +properties) — the smallest correct read surface that exercises +`execute_hedged()`, region pinning, cancellation, and the PPCB feedback +loop end-to-end. The Collection Read is a point read that rides the same +`(OperationType = Read)` gate, so it needs no per-page machinery; the +remaining metadata reads (Database / Offer / Throughput / +PartitionKeyRange) stay in Phase 2. Phase 2 widens to feed-style +operations (Query / ReadMany / ChangeFeed), each hedged **per page**, +plus those remaining metadata reads. The exact integration with the +`FeedRange` abstraction is being co-designed with the feed-operation +spec (see §16). Writes are not in scope for any phase. --- @@ -605,8 +611,8 @@ skip hedging even on a multi-region account. > > | Phase | Allowed `ResourceType` set | > |---|---| -> | 1 (MVP) | `{Document}` for point reads only — enforced by an additional `OperationType` guard inside the predicate (reads only, no writes). | -> | 2 | Phase 1 set ∪ feed-style operations (Query / ReadMany / ChangeFeed — still `ResourceType.Document` but `OperationType` differs) ∪ `{Database, Container, Offer, Throughput}` (metadata reads). | +> | 1 (MVP) | `{Document, DocumentCollection}` for point reads only — the data-plane point read plus the Collection Read that resolves container properties. Enforced by an additional `OperationType` guard inside the predicate (reads only, no writes), so only `(Document, Read)` and `(DocumentCollection, Read)` are eligible. | +> | 2 | Phase 1 set ∪ feed-style operations (Query / ReadMany / ChangeFeed — still `ResourceType.Document` but `OperationType` differs) ∪ `{Database, Offer, Throughput, PartitionKeyRange}` (the remaining metadata reads). | > | Future | Phase 2 set ∪ `{StoredProcedure}` (sprocs only — triggers / UDFs are not standalone operations). | > > Phase 1 implementations should hard-code the allowed `OperationType` @@ -2393,7 +2399,12 @@ also transient, §14.1 applies. | `should_hedge_excluded_to_one_region` | Reads NOT eligible when `ExcludeRegions` leaves < 2 applicable read endpoints | | `should_hedge_no_preferred_regions` | NOT eligible when application-preferred-region list is empty | | `should_hedge_write_never` | Writes (Create / Replace / Upsert / Delete / Patch) NEVER hedged regardless of topology | -| `should_hedge_non_document` | Non-Document `ResourceType`s excluded in Phase 1 | +| `should_hedge_non_document` | Non-hedgeable `ResourceType`s (e.g. `Database`) excluded | +| `should_hedge_container_read_multi_region` | Collection Read (`DocumentCollection` + `Read`) eligible on a multi-region account — the metadata point read this phase adds | +| `should_hedge_container_read_single_region` | Collection Read NOT eligible on a single-region account | +| `should_hedge_container_feed_not_hedged` | `read_all_containers` (`DocumentCollection` + `ReadFeed`) NOT hedged — `ReadFeed` is not an allowed `OperationType` | +| `should_hedge_container_query_not_hedged` | `query_containers` (`DocumentCollection` + `Query`) NOT hedged | +| `should_hedge_document_readfeed_not_hedged` | Over-broadening guard: a user change-feed read (`Document` + `ReadFeed`) stays non-hedgeable even though `Document` is hedgeable | | `should_hedge_disabled_override` | Per-operation `AvailabilityStrategy::Disabled` overrides client-level hedging | | `is_final_result_success` | 200 → final | | `is_final_result_conflict` | 409 → final | @@ -2453,7 +2464,7 @@ against the §1 Goals. | §1 Goal | Phase that closes it | |---|---| -| **G1. Reduce tail latency** (p99/p99.9 bounded by `threshold + RTT`) | Phase 1 (point reads). Phase 2 widens to feed-style operations + metadata. | +| **G1. Reduce tail latency** (p99/p99.9 bounded by `threshold + RTT`) | Phase 1 (point reads: data-plane `GetItem` + the Collection Read metadata call). Phase 2 widens to feed-style operations + the remaining metadata reads. | | **G2. Transparent to application** (single `CosmosResponse`; opt-in diagnostics) | Phase 1 (`HedgeDiagnostics`, `DiagnosticsContext` integration). | | **G3. Configurable** (single `threshold` knob at client and per-operation levels; explicit opt-out) | Phase 1. | | **G4. Complementary to failover** (composes with PPAF/PPCB; feeds PPCB) | Phase 1 (lock-free `LocationStateStore` interaction §9.1 + PPCB feedback callsite §9.5). | @@ -2468,9 +2479,14 @@ remain out of scope for every phase below. **Operation rows from §1 covered (Phase 1 column):** - Document point reads (`GetItem`). +- Collection Read (`DocumentCollection` + `Read`) — the metadata point + read that resolves container properties. It rides the same + `(OperationType = Read)` gate as `GetItem`, so it hedges with no + per-page machinery. Writes are excluded by spec rule (§1 Non-Goals, §5.1 row 4). Feed-style -operations (Query / ReadMany / ChangeFeed) and metadata operations are +operations (Query / ReadMany / ChangeFeed) and the remaining metadata +operations (Database / Offer / Throughput / PartitionKeyRange) are deferred to Phase 2 because they require additional coordination — see that section. @@ -2478,7 +2494,8 @@ that section. - `HedgeThreshold`, `HedgingStrategy`, `AvailabilityStrategy` types (§4). - `should_hedge()` covering point reads (§5.1; phase-allowed - `ResourceType` = `{Document}` with `OperationType = Read`). + `ResourceType` = `{Document, DocumentCollection}` with + `OperationType = Read`). - `is_final_result()` (§7.1). - `execute_hedged()` (§6.4) extending the `OperationAction::Hedge` arm of `operation_pipeline.rs` STAGE 7, with: @@ -2524,11 +2541,13 @@ that section. test. **§1 Goals closed at end of Phase 1:** G2, G3, G4, G5, G6 in full; -G1 for point reads only. +G1 for point reads only (data-plane point reads + the Collection Read +metadata call). **Out of scope this phase (deferred to Phase 2 / Future per §1 table):** -Feed-style operations (Query / ReadMany / ChangeFeed), metadata -operations, stored procedure execution, adaptive threshold tuning. +Feed-style operations (Query / ReadMany / ChangeFeed), the remaining +metadata reads (Database / Offer / Throughput / PartitionKeyRange), +stored procedure execution, adaptive threshold tuning. **Deliverables:** @@ -2549,8 +2568,8 @@ operations, stored procedure execution, adaptive threshold tuning. - `QueryItems` — hedged **per page**. - `ReadMany` — hedged **per page**. - Change feed (`ReadFeed`) — hedged **per page**. -- Metadata operations: Database / Container / Offer / Throughput - **reads only**. +- Metadata operations: Database / Offer / Throughput / PartitionKeyRange + **reads only** (the Container Read landed early in Phase 1 — see §5.1). **Scope (deferred — design pass required before scheduling):** @@ -2565,7 +2584,13 @@ operations, stored procedure execution, adaptive threshold tuning. - **Metadata cache invalidation.** Hedged metadata reads must not produce stale-cache races when one region returns an older view than another; decide whether to prefer the latest `_etag` / - resource id or the fastest response. + resource id or the fastest response. The Phase 1 Collection Read is + exempt from this open question: the driver only consumes the + container `_rid` and partition-key definition from that payload, both + immutable for the container's lifetime, so fastest-response-wins + cannot surface a stale value that changes routing. The remaining + metadata reads (Offer / Throughput, mutable settings) still need this + decision before they are made hedgeable. - **Diagnostics caveat for multi-stage operations.** Query / ReadMany / ChangeFeed contact regions *before* the hedge dispatch starts (query plan fetches, partition-key-range cache loads, diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs index eae8e6fce9b..785e6831cf0 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs @@ -49,6 +49,12 @@ //! injected. Diagnostics show `was_hedge=true`, //! `total_requests_launched=2`, `response_region=WEST_US` — the spec's //! canonical "tail-latency cut" outcome. +//! * [`hedging_container_read_primary_slow`] — the metadata analogue of the +//! slow case: 800 ms delay on East US `MetadataReadContainer` (the Collection +//! Read, `DocumentCollection` + `Read`), threshold 100 ms. The alternate wins +//! against West US, proving the `HEDGEABLE_RESOURCE_TYPES` widen hedges the +//! container-properties read end-to-end, not just at the `should_hedge` unit +//! level. //! * [`hedging_read_primary_503`] — 500 ms delay + 503 injected on East US //! `ReadItem`, threshold 100 ms. Same primary-vs-alternate race shape as //! the slow case, but the primary's eventual result is a transient @@ -155,7 +161,7 @@ use azure_data_cosmos_driver::fault_injection::{ }; use azure_data_cosmos_driver::in_memory_emulator::WriteMode; use azure_data_cosmos_driver::models::{ - AccountReference, CosmosOperation, ItemReference, PartitionKey, + AccountReference, CosmosOperation, DatabaseReference, ItemReference, PartitionKey, }; use azure_data_cosmos_driver::options::{ AvailabilityStrategy, DriverOptions, EndToEndOperationLatencyPolicy, ExcludedRegions, @@ -306,6 +312,29 @@ async fn read_item_result( .map(|maybe| maybe.expect("read_item returns a response body")) } +/// Issues a Collection Read (`read_container_by_name` — `DocumentCollection` + +/// `Read`) against `driver` with the supplied `OperationOptions` and returns +/// the response's optional `HedgeDiagnostics`. This is the container-properties +/// metadata point read that this phase makes hedgeable. Unlike +/// [`read_item_hedge_diagnostics`], it issues the operation directly through +/// `execute_operation` (bypassing the container cache) so the hedge race on the +/// metadata read itself is observable via the returned diagnostics. +async fn read_container_hedge_diagnostics( + driver: &Arc, + op_options: OperationOptions, +) -> Option { + let db_ref = DatabaseReference::from_name(driver.account().clone(), DB_NAME); + let operation = CosmosOperation::read_container_by_name(db_ref, COLL_NAME); + + let response = driver + .execute_operation(operation, op_options) + .await + .expect("read_container succeeds") + .expect("read_container returns a response body"); + + response.diagnostics().hedge_diagnostics().cloned() +} + // ───────────────────────────────────────────────────────────────────────────── // Tests // ───────────────────────────────────────────────────────────────────────────── @@ -443,7 +472,76 @@ async fn hedging_read_primary_slow() { ); } -/// Spec §15.2 row 3 — *primary returns 503 (delayed), alternate wins*. +/// Collection Read hedging — *primary slow, alternate wins*. +/// +/// 800 ms delay (no error) on East US `MetadataReadContainer`, threshold +/// 100 ms. The container-properties read this phase makes hedgeable +/// (`DocumentCollection` + `Read`) is the metadata analogue of +/// [`hedging_read_primary_slow`]: the primary attempt is still in flight when +/// the threshold elapses, `execute_hedged` spawns the alternate against West +/// US, and the alternate wins by virtue of having no delay injected. This is +/// the end-to-end proof that widening `HEDGEABLE_RESOURCE_TYPES` to include +/// `DocumentCollection` actually hedges the metadata read through the pipeline — +/// not just that `should_hedge` returns `true` at the unit level. +#[tokio::test] +async fn hedging_container_read_primary_slow() { + let ctx = setup_multi_region(WriteMode::Single).await; + + let condition = FaultInjectionConditionBuilder::new() + .with_operation_type(FaultOperationType::MetadataReadContainer) + .with_region(Region::EAST_US) + .build(); + // Delay only — no error. The primary Collection Read eventually succeeds, + // but only after the alternate has long since won the race. + let result = FaultInjectionResultBuilder::new() + .with_delay(Duration::from_millis(800)) + .with_probability(1.0) + .build(); + let rule = Arc::new( + FaultInjectionRuleBuilder::new("hedging-east-us-container-delay", result) + .with_condition(condition) + .build(), + ); + let rules = vec![Arc::clone(&rule)]; + + let (driver, op_options) = make_hedging_driver(&ctx, Duration::from_millis(100), rules).await; + + let hedge_diag = read_container_hedge_diagnostics(&driver, op_options) + .await + .expect( + "slow primary Collection Read should cause the threshold to elapse \ + and `execute_hedged` to spawn the alternate — `HedgeDiagnostics` \ + must be attached (spec §10.1)", + ); + + assert_eq!( + hedge_diag.terminal_state(), + HedgeTerminalState::AlternateWon, + "alternate must win when the primary Collection Read is slow past \ + threshold → terminal state must classify as AlternateWon (spec \ + §10.1.1); diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.alternate_region(), + Some(&Region::WEST_US), + "primary + alternate should both have been launched; diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.response_region(), + Some(&Region::WEST_US), + "alternate (West US) should be the winning region; diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.primary_region(), + &Region::EAST_US, + "primary_region must record East US (the losing region); \ + diag={hedge_diag:?}", + ); + assert!( + rule.hit_count() >= 1, + "the East US container-read delay rule should have been applied at least once", + ); +} /// /// 500 ms delay + 503 injected on East US `ReadItem`, threshold 100 ms. /// The delay is load-bearing: an instant 503 would win the primary side From 62f85a4f16f54a8745f37070418fe6e7f7f0c199 Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Tue, 7 Jul 2026 15:11:16 -0700 Subject: [PATCH 4/4] Add "unhedged" to Cosmos cSpell ignore list The CI Analyze job runs cSpell over every file changed in the PR, which includes the driver CHANGELOG entry that describes container feed reads and writes as staying "unhedged". "unhedged" is a domain term (the sibling "hedgeable" is already ignored) but was missing from the ignore list, so the spell-check task failed. Added it to sdk/cosmos/.cspell.json ignoreWords in alphabetical order. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/cosmos/.cspell.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index 41d961ad6b2..a777f6d44f1 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -287,6 +287,7 @@ "undrained", "unfaulted", "ungoverned", + "unhedged", "unparseable", "unrenamed", "unroutable",