diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index 41d961ad6b2..a777f6d44f1 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -287,6 +287,7 @@ "undrained", "unfaulted", "ungoverned", + "unhedged", "unparseable", "unrenamed", "unroutable", diff --git a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md index 1f6baeb657f..56f9a2f2f09 100644 --- a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - Added `TlsBackend` (re-exported) and a `tls_backend` option on `ConnectionPoolOptions` (`ConnectionPoolOptionsBuilder::with_tls_backend`), defaulting to `TlsBackend::Rustls`, available under the `rustls` feature, to pin the TLS backend used by the transport. This is additive and changes no behavior for the default (rustls) build; it only has an effect in builds that compile in multiple reqwest TLS backends, where reqwest would otherwise default to native-tls and the driver now pins rustls instead. ([#4649](https://github.com/Azure/azure-sdk-for-rust/pull/4649)) +- Cross-region read hedging now also covers the Collection Read metadata call (resolving container properties), in addition to data-plane point reads. When hedging is enabled (the default for multi-region accounts), the container-properties read is speculatively dispatched to a second preferred region after the threshold, reducing tail latency on the first operation against a container and on cache refreshes. No API change; enable or disable it via the existing `AvailabilityStrategy` / `AZURE_COSMOS_HEDGING_ENABLED` controls. ([#4710](https://github.com/Azure/azure-sdk-for-rust/pull/4710)) ### Breaking Changes diff --git a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md index 7c342ffd199..e05bcd75a7c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - Added `TlsBackend` (currently `TlsBackend::Rustls`, the default) and a `tls_backend` option on `ConnectionPoolOptions` (`ConnectionPoolOptionsBuilder::with_tls_backend` / `ConnectionPoolOptions::tls_backend`), available under the `rustls` feature. The driver asserts the selected backend on the `reqwest` transport, giving a supported way to pin the TLS backend without direct transport access. This is additive and changes no behavior for the default (rustls-only) build, where reqwest already negotiates rustls; it only has an effect in builds that compile in multiple reqwest TLS backends (e.g. `rustls` plus `native_tls`, absent reqwest's `http3` feature), where reqwest would otherwise default to native-tls and the driver now pins rustls instead. ([#4649](https://github.com/Azure/azure-sdk-for-rust/pull/4649)) +- Extended cross-region read hedging to the Collection Read metadata call (container properties). When hedging is enabled (the default for multi-region accounts), the driver now speculatively dispatches the container-properties read to a second preferred region after the threshold elapses, trimming the long tail on cold-start cache population and steady-state refresh. Scoped precisely to `(DocumentCollection, Read)`: container feed reads (`read_all_containers`), queries, and writes stay unhedged, and user change-feed / cross-partition feed reads (`Document` + `ReadFeed`) are unaffected. The PartitionKeyRange routing-map `ReadFeed` remains a separate future phase. ([#4710](https://github.com/Azure/azure-sdk-for-rust/pull/4710)) ### Breaking Changes diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md index 597ad4bda82..751cd01eacc 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/HEDGING_SPEC.md @@ -94,7 +94,8 @@ design review. | Queries (`QueryItems`) — page-level | ❌ | ✅ | ✅ | | `ReadMany` — page-level | ❌ | ✅ | ✅ | | Change feed — page-level | ❌ | ✅ | ✅ | -| Metadata operations (Database / Container / Offer / Throughput) | ❌ | ✅ | ✅ | +| Container Read (properties) — point read (`DocumentCollection` + `Read`) | ✅ | ✅ | ✅ | +| Other metadata reads (Database / Offer / Throughput / PartitionKeyRange) | ❌ | ✅ | ✅ | | Document writes (Create/Replace/Upsert/Delete/Patch) — any topology | ❌ | ❌ | ❌ | | Stored procedure execution (`ExecuteJavaScript`) | ❌ | ❌ | 🟡 candidate | @@ -104,13 +105,18 @@ design review. > procedure execution is a standalone server-side execution and > deferred to Future. -Phase 1 ships document point reads only — the smallest correct surface -that exercises `execute_hedged()`, region pinning, cancellation, and the -PPCB feedback loop end-to-end. Phase 2 widens to feed-style operations -(Query / ReadMany / ChangeFeed), each hedged **per page**, plus -metadata operations. The exact integration with the `FeedRange` -abstraction is being co-designed with the feed-operation spec (see -§16). Writes are not in scope for any phase. +Phase 1 ships document point reads plus the Collection Read metadata +call (`DocumentCollection` + `Read`, i.e. resolving container +properties) — the smallest correct read surface that exercises +`execute_hedged()`, region pinning, cancellation, and the PPCB feedback +loop end-to-end. The Collection Read is a point read that rides the same +`(OperationType = Read)` gate, so it needs no per-page machinery; the +remaining metadata reads (Database / Offer / Throughput / +PartitionKeyRange) stay in Phase 2. Phase 2 widens to feed-style +operations (Query / ReadMany / ChangeFeed), each hedged **per page**, +plus those remaining metadata reads. The exact integration with the +`FeedRange` abstraction is being co-designed with the feed-operation +spec (see §16). Writes are not in scope for any phase. --- @@ -605,8 +611,8 @@ skip hedging even on a multi-region account. > > | Phase | Allowed `ResourceType` set | > |---|---| -> | 1 (MVP) | `{Document}` for point reads only — enforced by an additional `OperationType` guard inside the predicate (reads only, no writes). | -> | 2 | Phase 1 set ∪ feed-style operations (Query / ReadMany / ChangeFeed — still `ResourceType.Document` but `OperationType` differs) ∪ `{Database, Container, Offer, Throughput}` (metadata reads). | +> | 1 (MVP) | `{Document, DocumentCollection}` for point reads only — the data-plane point read plus the Collection Read that resolves container properties. Enforced by an additional `OperationType` guard inside the predicate (reads only, no writes), so only `(Document, Read)` and `(DocumentCollection, Read)` are eligible. | +> | 2 | Phase 1 set ∪ feed-style operations (Query / ReadMany / ChangeFeed — still `ResourceType.Document` but `OperationType` differs) ∪ `{Database, Offer, Throughput, PartitionKeyRange}` (the remaining metadata reads). | > | Future | Phase 2 set ∪ `{StoredProcedure}` (sprocs only — triggers / UDFs are not standalone operations). | > > Phase 1 implementations should hard-code the allowed `OperationType` @@ -2393,7 +2399,12 @@ also transient, §14.1 applies. | `should_hedge_excluded_to_one_region` | Reads NOT eligible when `ExcludeRegions` leaves < 2 applicable read endpoints | | `should_hedge_no_preferred_regions` | NOT eligible when application-preferred-region list is empty | | `should_hedge_write_never` | Writes (Create / Replace / Upsert / Delete / Patch) NEVER hedged regardless of topology | -| `should_hedge_non_document` | Non-Document `ResourceType`s excluded in Phase 1 | +| `should_hedge_non_document` | Non-hedgeable `ResourceType`s (e.g. `Database`) excluded | +| `should_hedge_container_read_multi_region` | Collection Read (`DocumentCollection` + `Read`) eligible on a multi-region account — the metadata point read this phase adds | +| `should_hedge_container_read_single_region` | Collection Read NOT eligible on a single-region account | +| `should_hedge_container_feed_not_hedged` | `read_all_containers` (`DocumentCollection` + `ReadFeed`) NOT hedged — `ReadFeed` is not an allowed `OperationType` | +| `should_hedge_container_query_not_hedged` | `query_containers` (`DocumentCollection` + `Query`) NOT hedged | +| `should_hedge_document_readfeed_not_hedged` | Over-broadening guard: a user change-feed read (`Document` + `ReadFeed`) stays non-hedgeable even though `Document` is hedgeable | | `should_hedge_disabled_override` | Per-operation `AvailabilityStrategy::Disabled` overrides client-level hedging | | `is_final_result_success` | 200 → final | | `is_final_result_conflict` | 409 → final | @@ -2453,7 +2464,7 @@ against the §1 Goals. | §1 Goal | Phase that closes it | |---|---| -| **G1. Reduce tail latency** (p99/p99.9 bounded by `threshold + RTT`) | Phase 1 (point reads). Phase 2 widens to feed-style operations + metadata. | +| **G1. Reduce tail latency** (p99/p99.9 bounded by `threshold + RTT`) | Phase 1 (point reads: data-plane `GetItem` + the Collection Read metadata call). Phase 2 widens to feed-style operations + the remaining metadata reads. | | **G2. Transparent to application** (single `CosmosResponse`; opt-in diagnostics) | Phase 1 (`HedgeDiagnostics`, `DiagnosticsContext` integration). | | **G3. Configurable** (single `threshold` knob at client and per-operation levels; explicit opt-out) | Phase 1. | | **G4. Complementary to failover** (composes with PPAF/PPCB; feeds PPCB) | Phase 1 (lock-free `LocationStateStore` interaction §9.1 + PPCB feedback callsite §9.5). | @@ -2468,9 +2479,14 @@ remain out of scope for every phase below. **Operation rows from §1 covered (Phase 1 column):** - Document point reads (`GetItem`). +- Collection Read (`DocumentCollection` + `Read`) — the metadata point + read that resolves container properties. It rides the same + `(OperationType = Read)` gate as `GetItem`, so it hedges with no + per-page machinery. Writes are excluded by spec rule (§1 Non-Goals, §5.1 row 4). Feed-style -operations (Query / ReadMany / ChangeFeed) and metadata operations are +operations (Query / ReadMany / ChangeFeed) and the remaining metadata +operations (Database / Offer / Throughput / PartitionKeyRange) are deferred to Phase 2 because they require additional coordination — see that section. @@ -2478,7 +2494,8 @@ that section. - `HedgeThreshold`, `HedgingStrategy`, `AvailabilityStrategy` types (§4). - `should_hedge()` covering point reads (§5.1; phase-allowed - `ResourceType` = `{Document}` with `OperationType = Read`). + `ResourceType` = `{Document, DocumentCollection}` with + `OperationType = Read`). - `is_final_result()` (§7.1). - `execute_hedged()` (§6.4) extending the `OperationAction::Hedge` arm of `operation_pipeline.rs` STAGE 7, with: @@ -2524,11 +2541,13 @@ that section. test. **§1 Goals closed at end of Phase 1:** G2, G3, G4, G5, G6 in full; -G1 for point reads only. +G1 for point reads only (data-plane point reads + the Collection Read +metadata call). **Out of scope this phase (deferred to Phase 2 / Future per §1 table):** -Feed-style operations (Query / ReadMany / ChangeFeed), metadata -operations, stored procedure execution, adaptive threshold tuning. +Feed-style operations (Query / ReadMany / ChangeFeed), the remaining +metadata reads (Database / Offer / Throughput / PartitionKeyRange), +stored procedure execution, adaptive threshold tuning. **Deliverables:** @@ -2549,8 +2568,8 @@ operations, stored procedure execution, adaptive threshold tuning. - `QueryItems` — hedged **per page**. - `ReadMany` — hedged **per page**. - Change feed (`ReadFeed`) — hedged **per page**. -- Metadata operations: Database / Container / Offer / Throughput - **reads only**. +- Metadata operations: Database / Offer / Throughput / PartitionKeyRange + **reads only** (the Container Read landed early in Phase 1 — see §5.1). **Scope (deferred — design pass required before scheduling):** @@ -2565,7 +2584,13 @@ operations, stored procedure execution, adaptive threshold tuning. - **Metadata cache invalidation.** Hedged metadata reads must not produce stale-cache races when one region returns an older view than another; decide whether to prefer the latest `_etag` / - resource id or the fastest response. + resource id or the fastest response. The Phase 1 Collection Read is + exempt from this open question: the driver only consumes the + container `_rid` and partition-key definition from that payload, both + immutable for the container's lifetime, so fastest-response-wins + cannot surface a stale value that changes routing. The remaining + metadata reads (Offer / Throughput, mutable settings) still need this + decision before they are made hedgeable. - **Diagnostics caveat for multi-stage operations.** Query / ReadMany / ChangeFeed contact regions *before* the hedge dispatch starts (query plan fetches, partition-key-range cache loads, diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs index 9fab10e0f7e..be1aa43b0e7 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/hedging_eligibility.rs @@ -30,16 +30,40 @@ use crate::{ /// this constant is the upper bound. const DEFAULT_THRESHOLD_CAP: Duration = Duration::from_millis(1000); -/// Resource types eligible for cross-region hedging in the current phase. +/// Resource types eligible for cross-region hedging. /// -/// Subsequent phases widen this single constant — no other change to -/// [`should_hedge`] is required. -const HEDGEABLE_RESOURCE_TYPES: &[ResourceType] = &[ResourceType::Document]; - -/// Operation types eligible for cross-region hedging in the current phase. +/// - [`ResourceType::Document`] — data-plane point reads. +/// - [`ResourceType::DocumentCollection`] — the Collection Read that resolves +/// container properties. This metadata read sits on the critical path of +/// nearly every operation (cold-start cache population and steady-state +/// refresh), so hedging it trims the long tail when the primary region is +/// briefly slow. +/// +/// This pairs with [`HEDGEABLE_OPERATION_TYPES`] = `[Read]`: the only eligible +/// `(resource, operation)` tuples are therefore `(Document, Read)` and +/// `(DocumentCollection, Read)`. Container **writes** (`Create` / `Delete`) are +/// excluded by the `is_read_only()` guard, and the container **feed** reads +/// (`read_all_containers` → `ReadFeed`, `query_containers` → `Query`) are +/// excluded because their operation types are not in [`HEDGEABLE_OPERATION_TYPES`]. +/// +/// The other critical-path metadata read — the PartitionKeyRange `ReadFeed` +/// that builds the routing map — is intentionally **not** hedged here: making +/// it eligible would require adding `ReadFeed` to [`HEDGEABLE_OPERATION_TYPES`], +/// which (because `Document` is already hedgeable) would also make user +/// change-feed / cross-partition feed reads (`Document` + `ReadFeed`) eligible. +/// Scoping that correctly needs `(resource, operation)` tuple gating plus a +/// first-page-only / winning-region continuation-pinning story, and is left to +/// a follow-up. +const HEDGEABLE_RESOURCE_TYPES: &[ResourceType] = + &[ResourceType::Document, ResourceType::DocumentCollection]; + +/// Operation types eligible for cross-region hedging. /// -/// Future phases will append feed-style operations -/// (`Query` / `ReadFeed` / `QueryPlan`) and metadata reads. +/// Restricted to `Read` (point reads) so that only the point-read metadata and +/// data-plane paths hedge. Feed-style operations (`ReadFeed` / `Query` / +/// `QueryPlan`) are handled in a future phase — see the note on +/// [`HEDGEABLE_RESOURCE_TYPES`] about why widening this to `ReadFeed` requires +/// tuple gating to avoid unintentionally hedging user feed reads. const HEDGEABLE_OPERATION_TYPES: &[OperationType] = &[OperationType::Read]; /// Returns `true` when the operation is eligible for cross-region hedging. @@ -356,6 +380,39 @@ mod tests { CosmosOperation::read_database(db) } + fn db_reference() -> DatabaseReference { + let account = AccountReference::with_master_key( + Url::parse("https://acct.documents.azure.com/").unwrap(), + "k", + ); + DatabaseReference::from_name(account, "db") + } + + /// Collection Read (`DocumentCollection` + `Read`) — the metadata read this + /// phase makes hedgeable. + fn read_container_operation() -> CosmosOperation { + CosmosOperation::read_container_by_name(db_reference(), "c") + } + + /// `read_all_containers` (`DocumentCollection` + `ReadFeed`) — a container + /// *feed* read that must NOT be hedged. + fn read_all_containers_operation() -> CosmosOperation { + CosmosOperation::read_all_containers(db_reference()) + } + + /// `query_containers` (`DocumentCollection` + `Query`) — must NOT be hedged. + fn query_containers_operation() -> CosmosOperation { + CosmosOperation::query_containers(db_reference()) + } + + /// User change-feed read (`Document` + `ReadFeed`) — must NOT be hedged. + /// This guards against the over-broadening trap: widening resource types to + /// include `DocumentCollection` must not make any `ReadFeed` eligible. + fn read_all_items_operation() -> CosmosOperation { + let container = fake_container_reference(); + CosmosOperation::read_all_items(container, PartitionKey::from("pk")) + } + fn enabled_strategy() -> HedgingStrategy { HedgingStrategy::new(HedgeThreshold::new(Duration::from_millis(500)).unwrap()) } @@ -413,12 +470,55 @@ mod tests { #[test] fn should_hedge_non_document() { - // Reads against non-Document resource types are excluded in Phase 1. + // Reads against non-hedgeable resource types (e.g. Database) are excluded. let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); let op = read_database_operation(); assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); } + #[test] + fn should_hedge_container_read_multi_region() { + // Collection Read (DocumentCollection + Read) is hedgeable: it is a + // point read of container properties on the metadata critical path. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_container_operation(); + assert!(should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_read_single_region() { + let state = account_state_with_regions(&[Region::EAST_US]); + let op = read_container_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_feed_not_hedged() { + // read_all_containers is DocumentCollection + ReadFeed — a feed read, + // NOT eligible (ReadFeed is not in HEDGEABLE_OPERATION_TYPES). + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_all_containers_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_container_query_not_hedged() { + // query_containers is DocumentCollection + Query — NOT eligible. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = query_containers_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + + #[test] + fn should_hedge_document_readfeed_not_hedged() { + // Over-broadening guard: a user change-feed read (Document + ReadFeed) + // must remain non-hedgeable even though Document is a hedgeable resource + // type — because ReadFeed is not in HEDGEABLE_OPERATION_TYPES. + let state = account_state_with_regions(&[Region::EAST_US, Region::WEST_US_2]); + let op = read_all_items_operation(); + assert!(!should_hedge(Some(&enabled_strategy()), &op, &state, &[],)); + } + #[test] fn should_hedge_disabled_override() { // `None` represents Disabled at any layer — short-circuits before diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs index eae8e6fce9b..785e6831cf0 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/hedging.rs @@ -49,6 +49,12 @@ //! injected. Diagnostics show `was_hedge=true`, //! `total_requests_launched=2`, `response_region=WEST_US` — the spec's //! canonical "tail-latency cut" outcome. +//! * [`hedging_container_read_primary_slow`] — the metadata analogue of the +//! slow case: 800 ms delay on East US `MetadataReadContainer` (the Collection +//! Read, `DocumentCollection` + `Read`), threshold 100 ms. The alternate wins +//! against West US, proving the `HEDGEABLE_RESOURCE_TYPES` widen hedges the +//! container-properties read end-to-end, not just at the `should_hedge` unit +//! level. //! * [`hedging_read_primary_503`] — 500 ms delay + 503 injected on East US //! `ReadItem`, threshold 100 ms. Same primary-vs-alternate race shape as //! the slow case, but the primary's eventual result is a transient @@ -155,7 +161,7 @@ use azure_data_cosmos_driver::fault_injection::{ }; use azure_data_cosmos_driver::in_memory_emulator::WriteMode; use azure_data_cosmos_driver::models::{ - AccountReference, CosmosOperation, ItemReference, PartitionKey, + AccountReference, CosmosOperation, DatabaseReference, ItemReference, PartitionKey, }; use azure_data_cosmos_driver::options::{ AvailabilityStrategy, DriverOptions, EndToEndOperationLatencyPolicy, ExcludedRegions, @@ -306,6 +312,29 @@ async fn read_item_result( .map(|maybe| maybe.expect("read_item returns a response body")) } +/// Issues a Collection Read (`read_container_by_name` — `DocumentCollection` + +/// `Read`) against `driver` with the supplied `OperationOptions` and returns +/// the response's optional `HedgeDiagnostics`. This is the container-properties +/// metadata point read that this phase makes hedgeable. Unlike +/// [`read_item_hedge_diagnostics`], it issues the operation directly through +/// `execute_operation` (bypassing the container cache) so the hedge race on the +/// metadata read itself is observable via the returned diagnostics. +async fn read_container_hedge_diagnostics( + driver: &Arc, + op_options: OperationOptions, +) -> Option { + let db_ref = DatabaseReference::from_name(driver.account().clone(), DB_NAME); + let operation = CosmosOperation::read_container_by_name(db_ref, COLL_NAME); + + let response = driver + .execute_operation(operation, op_options) + .await + .expect("read_container succeeds") + .expect("read_container returns a response body"); + + response.diagnostics().hedge_diagnostics().cloned() +} + // ───────────────────────────────────────────────────────────────────────────── // Tests // ───────────────────────────────────────────────────────────────────────────── @@ -443,7 +472,76 @@ async fn hedging_read_primary_slow() { ); } -/// Spec §15.2 row 3 — *primary returns 503 (delayed), alternate wins*. +/// Collection Read hedging — *primary slow, alternate wins*. +/// +/// 800 ms delay (no error) on East US `MetadataReadContainer`, threshold +/// 100 ms. The container-properties read this phase makes hedgeable +/// (`DocumentCollection` + `Read`) is the metadata analogue of +/// [`hedging_read_primary_slow`]: the primary attempt is still in flight when +/// the threshold elapses, `execute_hedged` spawns the alternate against West +/// US, and the alternate wins by virtue of having no delay injected. This is +/// the end-to-end proof that widening `HEDGEABLE_RESOURCE_TYPES` to include +/// `DocumentCollection` actually hedges the metadata read through the pipeline — +/// not just that `should_hedge` returns `true` at the unit level. +#[tokio::test] +async fn hedging_container_read_primary_slow() { + let ctx = setup_multi_region(WriteMode::Single).await; + + let condition = FaultInjectionConditionBuilder::new() + .with_operation_type(FaultOperationType::MetadataReadContainer) + .with_region(Region::EAST_US) + .build(); + // Delay only — no error. The primary Collection Read eventually succeeds, + // but only after the alternate has long since won the race. + let result = FaultInjectionResultBuilder::new() + .with_delay(Duration::from_millis(800)) + .with_probability(1.0) + .build(); + let rule = Arc::new( + FaultInjectionRuleBuilder::new("hedging-east-us-container-delay", result) + .with_condition(condition) + .build(), + ); + let rules = vec![Arc::clone(&rule)]; + + let (driver, op_options) = make_hedging_driver(&ctx, Duration::from_millis(100), rules).await; + + let hedge_diag = read_container_hedge_diagnostics(&driver, op_options) + .await + .expect( + "slow primary Collection Read should cause the threshold to elapse \ + and `execute_hedged` to spawn the alternate — `HedgeDiagnostics` \ + must be attached (spec §10.1)", + ); + + assert_eq!( + hedge_diag.terminal_state(), + HedgeTerminalState::AlternateWon, + "alternate must win when the primary Collection Read is slow past \ + threshold → terminal state must classify as AlternateWon (spec \ + §10.1.1); diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.alternate_region(), + Some(&Region::WEST_US), + "primary + alternate should both have been launched; diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.response_region(), + Some(&Region::WEST_US), + "alternate (West US) should be the winning region; diag={hedge_diag:?}", + ); + assert_eq!( + hedge_diag.primary_region(), + &Region::EAST_US, + "primary_region must record East US (the losing region); \ + diag={hedge_diag:?}", + ); + assert!( + rule.hit_count() >= 1, + "the East US container-read delay rule should have been applied at least once", + ); +} /// /// 500 ms delay + 503 injected on East US `ReadItem`, threshold 100 ms. /// The delay is load-bearing: an instant 503 would win the primary side