From 65bc4b9a1570863c7b4c064b4a67e69609d73df7 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 12:24:36 +0000 Subject: [PATCH 01/44] Plan for in-memory-emulator with its own http(s) endpoint --- sdk/cosmos/.cspell.json | 1 + .../docs/adr/001_separate_host_crate.md | 35 ++ .../docs/adr/002_port_per_region.md | 34 ++ .../docs/adr/003_cleartext_http2.md | 37 ++ .../docs/adr/004_management_rest_api.md | 35 ++ .../docs/adr/005_json_config_startup_seed.md | 39 ++ .../docs/adr/006_gateway_v2_rntbd.md | 43 ++ .../007_region_offline_failover_primitives.md | 41 ++ .../docs/adr/008_defer_auth_https.md | 36 ++ .../docs/adr/009_ci_reuse_existing_suites.md | 39 ++ .../docs/adr/010_pr_sequencing.md | 39 ++ .../azure_data_cosmos_emulator/docs/plan.md | 493 ++++++++++++++++++ 12 files changed, 872 insertions(+) create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index e800dbcccc2..ab818543976 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -147,6 +147,7 @@ "IMDS", "inclusivity", "inlines", + "inmemory", "INVALIDARG", "isquery", "japaneast", diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md new file mode 100644 index 00000000000..98211d41c61 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md @@ -0,0 +1,35 @@ +# ADR-001 — Host in a separate binary crate; keep the emulator in the driver + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The in-memory emulator lives in `azure_data_cosmos_driver` and depends heavily on +driver-internal APIs (store, dispatch, EPK routing, RNTBD codec). It must become usable by other +SDKs behind a network port, but the implementation cannot be moved out of the driver without +exposing large swaths of internal surface. + +## Decision + +Add a new `publish = false` binary crate `azure_data_cosmos_emulator` that hosts the emulator. +The emulator implementation stays in the driver; the driver exposes a small, additional **public** +surface behind a feature flag (`__internal_in_memory_emulator_host`) that the host crate enables +automatically through its dependency declaration. + +## Consequences + +The host crate stays thin (CLI, HTTP listeners, config, management API). The emulator keeps full +access to driver internals. The extra surface is opt-in and clearly non-SemVer (the `__internal_` +prefix), so stable builds and the public API are unaffected. + +## Alternatives + +- Extracting the emulator into its own library crate was rejected: it would force a large, + unstable slice of driver internals to become public. +- Growing the existing test-only feature to cover hosting was rejected: it would entangle + in-process test wiring with server-only concerns. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md new file mode 100644 index 00000000000..519673e81d2 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md @@ -0,0 +1,34 @@ +# ADR-002 — Model each region as a distinct localhost port + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The driver reads account topology and routes subsequent requests to each region's +`databaseAccountEndpoint`, which must be an independently reachable network endpoint. The store +already resolves a request's region by `(scheme, host, port)`. + +## Decision + +Bind one gateway listener per region on a distinct `127.0.0.1:{port}`. A single shared +`EmulatorStore` backs every listener; the region is resolved per request from the `Host` header. +A single-region account is simply one port. Gateway V2 adds an optional second (thin-client) port +per region. + +## Consequences + +Multi-region topologies map cleanly onto the driver's existing endpoint routing with no new +region-resolution mechanism. The store models all regions internally, so one process and one store +serve every port. Region gateway URLs in the config are plain `http://127.0.0.1:{port}` values. + +## Alternatives + +- A single port differentiated by `Host` header (e.g. `eastus.localhost`) was rejected: it relies + on client-side DNS/hosts trickery and does not match how the driver dials distinct endpoints. +- One process per region was rejected: it would fragment the shared store and complicate + cross-region replication and failover simulation. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md new file mode 100644 index 00000000000..5e09851a6cf --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md @@ -0,0 +1,37 @@ +# ADR-003 — Serve cleartext HTTP/2 (h2c) and reuse the existing probe + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +HTTP/2 is a hard requirement for Gateway V2. PR1 hosts plaintext HTTP (TLS is deferred), so the +data plane must run cleartext HTTP/2 (h2c). The question was whether a driver change is needed to +negotiate h2c. + +## Decision + +Serve h2c from the host using `axum::serve` (hyper-util's `auto` builder accepts HTTP/2 +prior-knowledge connections automatically) and rely on the driver's **existing** behavior: +the `Http2Only` reqwest client already sets `http2_prior_knowledge()` (h2c against `http://`), +initialization already probes HTTP/2 then falls back to HTTP/1.1, and `http://` is already +permitted for loopback emulator hosts. No mandatory driver change. + +## Consequences + +The hosted emulator negotiates HTTP/2 with the unmodified driver. A change to +`has_explicit_http2_incompatibility` is held as a **contingency**, applied only if end-to-end +validation reveals the driver does not cleanly fall back from h2c to HTTP/1.1 against a +cleartext HTTP/1.1-only server. + +## Alternatives + +- Serving HTTP/1.1 in PR1 and deferring HTTP/2 to the TLS PR was rejected: it would leave the + Gateway V2 path without validation for longer and understate the HTTP/2 requirement. +- Adding a new client toggle to force prior knowledge was rejected as redundant with existing + behavior. + +## References + +- Plan & summary: ../plan.md +- Transport pipeline spec: ../../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md new file mode 100644 index 00000000000..07fcb2f1221 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md @@ -0,0 +1,35 @@ +# ADR-004 — Expose control-plane actions via a separate management REST API + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +Some control-plane actions are part of the Cosmos gateway contract (database/container/offer/item +CRUD, PK-ranges, account read) and are already served on the region gateway ports. Others — +partition split/merge, region offline/online, runtime write-region failover, per-partition +failover toggle, replication pause/resume — have **no** gateway equivalent and today exist only as +`EmulatorStore` method calls reachable in-process. + +## Decision + +Serve the emulator-only control-plane actions through a dedicated **management REST API** on its +own port, distinct from the Cosmos wire protocol. Gateway-native lifecycle operations are **not** +duplicated there; callers use the standard Cosmos endpoints (or startup config seeding) for those. + +## Consequences + +Other SDKs and operators drive emulator-specific behavior (split, merge, failover, offline) +over HTTP without an in-process handle. The Cosmos data plane stays a pure wire-protocol surface, +and the management API never collides with Cosmos paths because it lives on a separate port. + +## Alternatives + +- Overloading the Cosmos data-plane ports with a reserved path prefix (e.g. `/_emulator/...`) was + rejected as more collision-prone and less discoverable than a dedicated port. +- Duplicating database/container CRUD in the management API was rejected: those are already + expressible through the gateway contract. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md new file mode 100644 index 00000000000..5619de34e31 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md @@ -0,0 +1,39 @@ +# ADR-005 — Drive startup topology and seed data from a JSON config file + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The hosted emulator needs a way to declare account topology (regions, write mode, consistency, +replication), the databases and containers to create, and optional seed documents — applied on +startup, before any client connects. The driver's config types are not `serde`-friendly (some +fields hold closures and shared mutable state). + +## Decision + +Accept a single JSON file via `--config`. The host owns `serde` DTOs that mirror the config and +translate them into driver types (`VirtualAccountConfig`, `VirtualRegion`, `ContainerConfig`). +Seed documents are created through the normal write path — one synthesized create-item request per +item through `execute_request` — so EPK routing, RU accounting, and replication match +client-issued writes. The management REST API can further modify state at runtime. YAML is +deferred. + +## Consequences + +Startup provisioning is declarative and reproducible; runtime mutation stays available through the +control-plane API. Keeping the DTOs in the host crate leaves the driver's config types untouched. +JSON is supported first because `serde_json` is already a workspace dependency. + +## Alternatives + +- Adding `serde` derives directly to the driver config types was rejected: several fields are not + serializable, and it would leak host concerns into the driver. +- Shipping YAML in the first PR was rejected: `serde_yaml` is unmaintained; a maintained crate can + be added later if needed. +- Seeding items by writing store internals directly was rejected: routing through `execute_request` + guarantees identical semantics to real writes. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md new file mode 100644 index 00000000000..6427791ebb3 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md @@ -0,0 +1,43 @@ +# ADR-006 — Simulate Gateway V2 by promoting the inverse RNTBD codec, config-gated + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +Gateway V2 (thin-client) uses the RNTBD wire format over HTTP/2 instead of JSON REST. To let the +hosted emulator exercise the driver's Gateway V2 path, the emulator must act as an RNTBD **server**: +answer the connectivity probe, decode inbound request frames, and encode outbound response frames. +The driver already owns the **client** halves (`RntbdRequestFrame::write`, `RntbdResponse::read`); +the inverse halves exist only as `#[cfg(test)]` helpers. Today the emulator advertises no +thin-client endpoints, so the driver always suppresses Gateway V2 against it. + +## Decision + +Fold Gateway V2 simulation into PR1, gated by config. Promote the test-only inverse codec +(`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, **co-located** in +`src/driver/transport/rntbd/` behind the host feature, reusing the existing token and status +primitives. When a region declares a `thinClientPort`, the emulator binds a thin-client listener +that answers `POST /connectivity-probe` with `200`, decodes RNTBD + thin-client headers into a +logical operation, dispatches through the shared store, and encodes an RNTBD response; the account +topology advertises `thinClient{Readable,Writable}Locations`. + +## Consequences + +Gateway V1 always works; Gateway V2 is opt-in per region, so CI can run the existing suites in both +modes. Because the request pipeline selects Gateway V2 purely from thin-client advertisement, no +driver routing change is needed. The codec halves stay symmetric and co-located for maximal reuse. + +## Alternatives + +- Shipping Gateway V2 as a separate later PR was rejected: hosting both from the start lets one CI + pass validate both protocol paths end-to-end. +- Placing the server-side codec in the emulator module was rejected in favor of co-location with + the client codec (Option A), which maximizes reuse and keeps the wire format in one place. +- Always advertising thin-client endpoints was rejected: config-gating keeps Gateway V1 as the + safe default. + +## References + +- Plan & summary: ../plan.md +- Gateway V2 spec: ../../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md new file mode 100644 index 00000000000..10875fa5676 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md @@ -0,0 +1,41 @@ +# ADR-007 — Add runtime region-offline and write-region failover primitives + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The control plane must support taking a region fully offline and changing the write region at +runtime, to exercise the driver's failover behavior. The store has replication pause/resume and a +per-partition failover flag, but **no** first-class region-offline state and **no** runtime +write-region selection — the write region is fixed to the first configured region in single-write +mode. + +## Decision + +Introduce two new store primitives in PR2, gated behind the host feature: + +- **Region offline/online:** a runtime-mutable set of offline regions. Offline regions are dropped + from the account topology's readable/writable locations and return `503` for data-plane requests. +- **Runtime write-region failover:** a runtime-mutable write-region override (an + `Arc>>`, mirroring the existing per-partition-failover atomic pattern) + consulted by `is_write_region` / `write_region_name` and by the writable-locations topology. + +Both ship with net-new in-process tests and corresponding management REST endpoints. + +## Consequences + +The emulator can reproduce region outage and planned/unplanned write-region failover, so the +driver's re-routing and topology-refresh paths are testable offline. The override defaults to +`None`, preserving today's "first region is the write region" behavior when unused. + +## Alternatives + +- Reusing replication pause/resume to approximate an offline region was rejected: it models lag, + not a region being unreachable, and does not remove the region from topology. +- Making these primitives part of PR1 was rejected: the existing suites do not need them, and they + warrant dedicated tests, so they belong in their own PR. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md new file mode 100644 index 00000000000..ee70baa53ad --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md @@ -0,0 +1,36 @@ +# ADR-008 — Defer HTTPS and authentication to a dedicated later PR + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +Cosmos DB supports key-based auth and Entra ID auth, and production endpoints require HTTPS. +Supporting TLS (certificate provisioning and trust) and authentication introduces real friction +that is orthogonal to standing up the hosted emulator and its control plane. + +## Decision + +PR1 and PR2 host **plaintext HTTP with no authentication**. A dedicated later PR (PR3) adds, +behind an `auth` config block and CLI flags: + +- Optional HTTPS via axum + rustls (`--https --cert --key`); h2c remains the default when off. +- Auth modes: `none` (default), `key` (validate the `Authorization` HMAC against a primary key and + a primary read-only key, matching the service), and `entra` (validate a bearer JWT and check its + object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). + +## Consequences + +The initial hosting and control-plane work ships without certificate or token friction, and can be +validated over h2c immediately. Auth and HTTPS land as a self-contained, reviewable increment. + +## Alternatives + +- Building auth and HTTPS into PR1 was rejected: it enlarges and slows the first deliverable and + couples unrelated concerns. +- Making auth mandatory was rejected: emulator use cases are predominantly local and offline, where + `none` over h2c is the least-friction default. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md new file mode 100644 index 00000000000..13fe0295659 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md @@ -0,0 +1,39 @@ +# ADR-009 — Validate via the existing suites over a new `emulator_inmemory` cfg + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The hosted emulator is only meaningful if it faithfully serves the real wire protocol. The most +convincing validation is the existing `emulator` / `emulator_vnext` integration suites passing +against it. The suites are gated at runtime by a `test_category` cfg set via `RUSTFLAGS`. + +## Decision + +Add a new `test_category = "emulator_inmemory"` cfg (registered in the `build.rs` of both +`azure_data_cosmos` and `azure_data_cosmos_driver`). Extend +`sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary with a +provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the +hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on +`Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway V2 +(thin-client) modes. This CI pass replaces a standalone HTTP/2 validation spike. + +## Consequences + +The same battle-tested suites validate h2c and RNTBD end-to-end, in both gateway modes, without +new bespoke tests. Starting non-blocking (`ContinueOnError`) surfaces failures as "succeeded with +issues" while the host stabilizes. + +## Alternatives + +- A separate standalone h2c validation spike was rejected: running the real suites over the hosted + emulator is stronger and is exactly what the combined PR delivers. +- A bespoke `emulator_inmemory`-only test set was rejected: reusing the existing suites maximizes + coverage and avoids drift. +- Reusing the `emulator_vnext` cfg was rejected: its behavioral-divergence skips do not match the + in-memory emulator, so a distinct cfg is cleaner. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md new file mode 100644 index 00000000000..6902962a43e --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md @@ -0,0 +1,39 @@ +# ADR-010 — PR sequencing + +**Status:** Accepted +**Date:** 2026-07-14 + +## Context + +The work spans a new crate, host feature surface, data-plane hosting for two gateway flavors, a +control-plane API, new store primitives, CI, and eventually auth/HTTPS. It needs to be split into +reviewable increments that each land something coherent. + +## Decision + +Sequence the work as three PRs: + +- **PR1** — the new host crate; per-region h2c hosting of Gateway V1 **and** Gateway V2 + (config-gated); the management REST API over the control-plane actions that map to existing store + methods; and CI running the existing suites against the hosted emulator in both gateway modes. +- **PR2** — the new store primitives (region offline/online, runtime write-region failover) with + net-new tests and their management endpoints. +- **PR3** — optional HTTPS and authentication. + +## Consequences + +PR1 is independently meaningful because the CI leg validates it end-to-end; it does not depend on +the new primitives, which the existing suites do not exercise. PR2 and PR3 are self-contained, +testable increments. Gateway V2 is folded into PR1 so a single CI pass validates both protocol +paths. + +## Alternatives + +- Splitting hosting and CI into separate PRs was rejected: hosting without the CI validation proves + nothing, so they ship together. +- Deferring Gateway V2 to its own PR was rejected: hosting both gateways from the start lets one CI + pass cover both (ADR-006). + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md new file mode 100644 index 00000000000..5fca0b00809 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -0,0 +1,493 @@ +# Hosted In-Memory Cosmos DB Emulator — Plan & Summary + +**Status:** Draft / Plan (not yet implemented) +**Date:** 2026-07-14 +**Crates:** `azure_data_cosmos_emulator` (new binary host), `azure_data_cosmos_driver` (emulator core, behind a feature) +**Feature gate:** `__internal_in_memory_emulator_host` (non-SemVer, host-only surface) + +--- + +## Table of Contents + +1. [Goals & Motivation](#1-goals--motivation) +2. [Scope, Phasing & Feature Gating](#2-scope-phasing--feature-gating) +3. [Architecture Overview](#3-architecture-overview) +4. [Configuration File](#4-configuration-file) +5. [Data-Plane Hosting (Gateway V1 & Gateway V2)](#5-data-plane-hosting-gateway-v1--gateway-v2) +6. [Management REST API (Control Plane)](#6-management-rest-api-control-plane) +7. [Rust Public API Surface](#7-rust-public-api-surface) +8. [HTTP/2 & Transport Notes](#8-http2--transport-notes) +9. [CI Integration](#9-ci-integration) +10. [Authentication & HTTPS (Deferred)](#10-authentication--https-deferred) +11. [Architecture Decision Records](#11-architecture-decision-records) +12. [Open Questions & Future Work](#12-open-questions--future-work) +13. [References](#13-references) + +--- + +## 1. Goals & Motivation + +The in-memory Cosmos DB emulator lives in `azure_data_cosmos_driver` and today can only be +injected **in-process** as an `HttpClient` for Rust-internal tests. It intercepts requests at +the `azure_core::http::HttpClient` boundary, so the whole operation pipeline (routing, session +management, retry, failover) executes normally while HTTP I/O is replaced by a deterministic +in-memory store. + +The goal of this work is to make the same emulator usable by **other language SDKs** (and by +any client) by hosting it **behind real network ports over HTTP/2**, without moving the +emulator implementation out of the driver crate (it depends heavily on driver-internal APIs). + +### Design principles + +- **Emulator core stays in the driver.** The host crate is a thin shell. The driver exposes + just enough **public** surface — gated behind a feature flag the host crate sets automatically + — to construct, seed, control, and serve the emulator. +- **Wire fidelity.** The hosted emulator speaks the real Cosmos DB wire protocol: Gateway V1 + (JSON REST) and, when enabled, Gateway V2 (thin-client / RNTBD over HTTP/2). Existing SDK test + suites run against it unchanged. +- **Deterministic, offline, no Docker.** No network access, no external accounts, no containers + required to run the store itself. +- **Control plane is first-class.** Emulator-specific operations that have no Cosmos gateway + equivalent (partition split/merge, region offline/online, runtime write-region failover, + per-partition failover toggle, replication pause/resume) are exposed as a **management REST + API** and can be applied both at startup (via config) and at runtime. + +--- + +## 2. Scope, Phasing & Feature Gating + +The work ships across three pull requests. + +| PR | Contents | Feature/cfg | +| --- | --- | --- | +| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway V2 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator_host`, `test_category = "emulator_inmemory"` | +| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | +| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | + +### Feature gating + +- The emulator core remains behind the existing `__internal_in_memory_emulator` feature. +- A new feature `__internal_in_memory_emulator_host = ["__internal_in_memory_emulator"]` gates + the additional **host-only** public surface (the server-side RNTBD codec halves and the new + control-plane primitives). The host crate enables it automatically via its dependency + declaration, so consumers never set it by hand. +- Both features use the `__internal_` prefix: this surface is **not** part of any SemVer + contract and may change at any time. + +--- + +## 3. Architecture Overview + +```text + ┌────────────────────────────────────────────────┐ + │ azure_data_cosmos_emulator (bin) │ + │ │ + client SDK ──h2c──▶ │ Region "East US" gateway listener :8081 ─┐ │ + (any lang) │ Region "East US" thin-client :8444 ─┤ │ + │ Region "West US" gateway listener :8082 ─┼──▶ │ Arc + operator ──http─▶ │ Region "West US" thin-client :8445 ─┤ │ └── Arc + (curl) │ Management API :9090 ─┘ │ + └────────────────────────────────────────────────┘ + (all listeners share ONE store) +``` + +- **Port-per-region.** Each virtual region is a distinct `127.0.0.1:{port}` gateway endpoint. + The driver reads account topology, then routes to each region's `databaseAccountEndpoint`, + which must be independently reachable. The store already resolves a request's region by + `(scheme, host, port)`, so a single shared store serves every port. +- **Single shared store.** One `InMemoryEmulatorHttpClient` / `EmulatorStore` backs all + listeners; region is resolved per request from the `Host` header. +- **Data-plane bridge.** Each gateway listener rebuilds an `azure_core::http::Request` from the + incoming HTTP request and calls `InMemoryEmulatorHttpClient::execute_request`, then converts + the `AsyncRawResponse` back to an HTTP response. +- **Gateway V2 (optional).** When a region declares a `thinClientPort`, the host binds a + thin-client listener that answers the connectivity probe and speaks RNTBD; the store's account + topology then advertises `thinClient{Readable,Writable}Locations`. +- **Management port.** A dedicated port serves the control-plane REST API, kept separate from the + Cosmos wire protocol so the two never collide. + +### Crate layout (proposed) + +```text +sdk/cosmos/azure_data_cosmos_emulator/ +├── Cargo.toml # publish = false; deps: clap, axum, tokio, serde, serde_json, tracing +├── README.md +├── docs/ +│ ├── plan.md # this document +│ └── adr/ # architecture decision records +└── src/ + ├── main.rs # CLI (clap), startup, listener wiring + ├── config.rs # serde DTOs + translation to driver types + seeding + ├── data_plane.rs # HTTP <-> azure_core::http::Request bridge (Gateway V1) + ├── gateway_v2.rs # thin-client listener, connectivity probe, RNTBD bridge + └── management.rs # control-plane REST API (axum router) +``` + +--- + +## 4. Configuration File + +A single JSON file, referenced with `--config`, describes the account topology, the databases +and containers to create, and optional seed items — all applied on startup. The management REST +API can further modify state at runtime. (YAML support is deferred; see ADR-005.) + +### 4.1 Sample + +```json +{ + "account": { + "id": "emulator-account", + "writeMode": "single", + "consistency": "Session", + "perPartitionFailover": false, + "throttling": false, + "regions": [ + { "name": "East US", "gatewayPort": 8081, "thinClientPort": 8444, "regionId": 0 }, + { "name": "West US", "gatewayPort": 8082, "thinClientPort": 8445, "regionId": 1 } + ], + "replication": { "minDelayMs": 20, "maxDelayMs": 50, "maxBufferedReplications": 10000 }, + "replicationOverrides": [ + { "source": "East US", "target": "West US", "minDelayMs": 200, "maxDelayMs": 500 } + ] + }, + "management": { "port": 9090 }, + "databases": [ + { + "id": "testdb", + "containers": [ + { + "id": "testcoll", + "partitionKey": { "paths": ["/pk"], "kind": "Hash", "version": 2 }, + "partitionCount": 4, + "throughput": 400, + "seedItems": [ + { "partitionKey": ["pk1"], "document": { "id": "1", "pk": "pk1", "value": 42 } }, + { "partitionKey": ["pk2"], "document": { "id": "2", "pk": "pk2", "value": 7 } } + ] + } + ] + } + ] +} +``` + +### 4.2 Field reference + +| Path | Type | Notes | +| --- | --- | --- | +| `account.writeMode` | `"single" \| "multi"` | Maps to `WriteMode`. In `single`, the first region is the hub/write region. | +| `account.consistency` | enum | `Strong \| BoundedStaleness \| Session \| ConsistentPrefix \| Eventual`. | +| `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | +| `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | +| `account.regions[].gatewayPort` | u16 | Gateway V1 (JSON REST) port. Region endpoint = `http://127.0.0.1:{gatewayPort}`. | +| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway V2 simulation for the region. | +| `account.regions[].regionId` | u64, optional | Auto-assigned by position when omitted. | +| `account.replication` | object | Default replication delay + buffer cap. | +| `account.replicationOverrides[]` | array | Per source→target replication overrides. | +| `management.port` | u16 | Control-plane REST API port. | +| `databases[].containers[].partitionKey` | object | Standard Cosmos partition key definition (`paths`, `kind`, `version`). | +| `databases[].containers[].partitionCount` | u32 | Initial physical partition count. | +| `databases[].containers[].throughput` | u32 | Provisioned RU/s (drives throttling when enabled). | +| `databases[].containers[].seedItems[]` | array | Documents created on startup; each carries its `partitionKey` value array and the `document` body. | + +Seed items are created through the same request path as real writes (a synthesized create-item +request per item), so EPK routing, RU accounting, and replication behave identically to +client-issued writes. + +--- + +## 5. Data-Plane Hosting (Gateway V1 & Gateway V2) + +### 5.1 Gateway V1 (JSON REST) + +The default. Each region's gateway listener bridges HTTP ⇄ `azure_core::http::Request` and +delegates to `execute_request`. All existing point operations and Cosmos control-plane +operations that are part of the gateway contract (database/container/offer CRUD, PK-ranges, +account read) are served here unchanged. + +### 5.2 Gateway V2 (thin-client / RNTBD) + +Enabled per region by setting `thinClientPort`. When enabled: + +1. The account topology advertises `thinClientReadableLocations` / `thinClientWritableLocations` + pointing at `http://127.0.0.1:{thinClientPort}`. +2. The thin-client listener answers `POST /connectivity-probe` with `200 OK`. +3. Data-plane requests arrive as RNTBD frames wrapped in HTTP/2 POSTs. The listener **decodes** + the RNTBD request frame + thin-client headers, reconstructs the logical operation, dispatches + it through the same store, and **encodes** the result as an RNTBD response frame. + +The driver already owns the client-side RNTBD codec (`RntbdRequestFrame::write`, +`RntbdResponse::read`). This work promotes the currently test-only inverse halves +(`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, co-located in +`src/driver/transport/rntbd/` behind the host feature (ADR-006). Because the driver's request +pipeline decides Gateway V2 purely from whether the endpoint advertises a thin-client URL, +advertising it plus serving RNTBD is sufficient — no driver routing change is required. + +--- + +## 6. Management REST API (Control Plane) + +The management API exposes only the control-plane actions that have **no Cosmos gateway +equivalent**. Database, container, offer, and item lifecycle are **not** here — those are served +by the standard Cosmos data-plane endpoints on the region gateway ports (or seeded via config). + +All endpoints below are served on the management port (e.g. `:9090`), use JSON bodies, and +return JSON. Errors use conventional HTTP status codes with a `{ "error": "..." }` body. + +### 6.1 Introspection + +```text +GET /health + → 200 { "status": "ok" } + +GET /account + → 200 { topology: regions, writeMode, consistency, offlineRegions, writeRegion, ... } +``` + +### 6.2 Partition split / merge + +```text +POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split + → 202 { "database": "testdb", "container": "testcoll", "parent": 0, "children": [4, 5] } + +POST /databases/{db}/containers/{coll}/partitions/merge + body: { "partitionIds": [4, 5] } + → 202 { "merged": [4, 5], "into": 6 } +``` + +Sample: + +```bash +curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split +``` + +### 6.3 Region offline / online (PR2) + +```text +POST /regions/{region}/offline + → 200 { "region": "West US", "state": "offline" } + +POST /regions/{region}/online + → 200 { "region": "West US", "state": "online" } +``` + +An offline region is dropped from the account topology's readable/writable locations and returns +`503` for data-plane requests, so the driver fails over exactly as it would against the service. + +Sample: + +```bash +curl -X POST http://127.0.0.1:9090/regions/West%20US/offline +``` + +### 6.4 Runtime write-region failover (PR2) + +```text +POST /failover + body: { "writeRegion": "West US" } + → 200 { "writeRegion": "West US" } +``` + +Changes which region accepts writes in single-write mode at runtime; the next account read +reflects the new `writableLocations`, and the driver re-routes writes accordingly. + +Sample: + +```bash +curl -X POST http://127.0.0.1:9090/failover \ + -H 'content-type: application/json' \ + -d '{ "writeRegion": "West US" }' +``` + +### 6.5 Per-partition failover (PPAF) toggle + +```text +PUT /config/per-partition-failover + body: { "enabled": true } + → 200 { "enabled": true } +``` + +### 6.6 Replication pause / resume + +```text +POST /regions/{region}/replication/pause + → 200 { "region": "West US", "replication": "paused" } + +POST /regions/{region}/replication/resume + → 200 { "region": "West US", "replication": "resumed" } +``` + +Pausing replication to a target region buffers replicas (up to the configured cap, then 429/3075), +which is the emulator's mechanism for simulating a lagging or partially-unavailable region without +taking it fully offline. + +--- + +## 7. Rust Public API Surface + +This is the surface the host crate consumes, all behind `__internal_in_memory_emulator_host`. +Items marked **(PR2)** are new primitives introduced in the second PR and are shown here as the +proposed signatures. + +### 7.1 Constructing and seeding the emulator + +```rust +use std::sync::Arc; +use url::Url; +use azure_data_cosmos_driver::in_memory_emulator::{ + ConsistencyLevel, ContainerConfig, InMemoryEmulatorHttpClient, VirtualAccountConfig, + VirtualRegion, WriteMode, +}; + +// Region gateway endpoints are plain http:// loopback ports (h2c). +let config = VirtualAccountConfig::new(vec![ + VirtualRegion::new("East US", Url::parse("http://127.0.0.1:8081")?), + VirtualRegion::new("West US", Url::parse("http://127.0.0.1:8082")?), +])? +.with_write_mode(WriteMode::Single) +.with_consistency(ConsistencyLevel::Session); + +let emulator = Arc::new(InMemoryEmulatorHttpClient::new(config)); +let store = emulator.store(); + +store.create_database("testdb"); +store.create_container_with_config( + "testdb", + "testcoll", + serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + }))?, + ContainerConfig::new().with_partition_count(4).with_throughput(400).build()?, +); +``` + +### 7.2 Data-plane bridge (host crate) + +```rust +// Every region gateway listener handler funnels here. +async fn serve_cosmos( + emulator: Arc, + incoming: http::Request, +) -> azure_core::Result> { + // Rebuild an azure_core request from method + Host + path/query + headers + body. + let cosmos_request = rebuild_azure_core_request(incoming).await?; + let raw = emulator.execute_request(&cosmos_request).await?; + to_http_response(raw).await +} +``` + +### 7.3 Control-plane primitives + +```rust +// Already public today (behind the base emulator feature): +store.split_partition("testdb", "testcoll", 0); +store.merge_partitions("testdb", "testcoll", &[4, 5]); +store.pause_replication("West US"); +store.resume_replication("West US"); +store.set_per_partition_failover(true); + +// (PR2) new primitives — proposed signatures: +store.set_region_offline("West US"); // drop from topology + 503 on that region +store.set_region_online("West US"); +store.set_write_region("West US")?; // runtime single-write failover; Err on unknown region +``` + +### 7.4 Gateway V2 server codec (behind host feature) + +```rust +// Promoted from test-only helpers to production, co-located with the client codec. +use azure_data_cosmos_driver::driver::transport::rntbd::{RntbdRequestFrame, RntbdResponse}; + +let frame = RntbdRequestFrame::read(&request_bytes)?; // decode inbound (server side) +// ... dispatch through the store ... +let mut out = Vec::new(); +response.write(&mut out)?; // encode outbound (server side) +``` + +--- + +## 8. HTTP/2 & Transport Notes + +HTTP/2 is a hard requirement for Gateway V2. The relevant driver behavior **already exists**: + +- The `Http2Only` reqwest client sets `http2_prior_knowledge()`, which performs cleartext h2c + against `http://` endpoints. +- Driver initialization probes HTTP/2 first and falls back to HTTP/1.1 on an explicit + incompatibility signal (`should_downgrade_http2` / `has_explicit_http2_incompatibility`). +- `ensure_endpoint_scheme_allowed` already permits `http://` for `localhost` / `127.0.0.1` / + `[::1]` (and `AZURE_COSMOS_EMULATOR_HOST`). + +`axum::serve` uses hyper-util's protocol-detecting `auto` builder, which accepts h2c +prior-knowledge connections automatically. So the host serves h2c, the existing probe negotiates +HTTP/2 against it, and no mandatory driver change is required. A targeted change to the +incompatibility matcher is a **contingency** only if end-to-end validation reveals a gap +(ADR-003). + +--- + +## 9. CI Integration + +- Register a new `test_category = "emulator_inmemory"` cfg in the `build.rs` of both + `azure_data_cosmos` and `azure_data_cosmos_driver`. +- Extend `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary + with a provisioning config, wait for `GET /health`, then set `AZURE_COSMOS_CONNECTION_STRING` + to the hosted endpoint and the `emulator_inmemory` cfg. +- Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` modeled on `Cosmos_vnext_emulator`, + running the **existing** emulator suites against the hosted emulator in **both** Gateway V1 and + Gateway V2 (thin-client) modes. This is the real end-to-end validation of h2c + RNTBD. + +--- + +## 10. Authentication & HTTPS (Deferred) + +PR1 and PR2 host **plaintext HTTP with no authentication**. PR3 adds, behind an `auth` config +block and CLI flags: + +- **HTTPS (optional):** `--https --cert --key ` (axum + rustls). When off, the host + stays on h2c. +- **Auth modes:** `none` (default), `key` (validate the `Authorization` HMAC against a primary + key and a primary read-only key, matching the service), and `entra` (validate a bearer JWT and + check its object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). + +Splitting auth/HTTPS into its own PR keeps certificate and token friction out of the initial +hosting and control-plane work (ADR-008). + +--- + +## 11. Architecture Decision Records + +| ADR | Decision | +| --- | --- | +| [ADR-001](adr/001_separate_host_crate.md) | Host in a separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. | +| [ADR-002](adr/002_port_per_region.md) | Model each region as a distinct localhost port; one shared store. | +| [ADR-003](adr/003_cleartext_http2.md) | Serve cleartext HTTP/2 (h2c) and reuse the driver's existing prior-knowledge probe. | +| [ADR-004](adr/004_management_rest_api.md) | Expose emulator-only control-plane actions via a separate management REST API. | +| [ADR-005](adr/005_json_config_startup_seed.md) | Drive startup topology and seed data from a JSON config file; defer YAML. | +| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway V2 by promoting the test-only inverse RNTBD codec to production, config-gated. | +| [ADR-007](adr/007_region_offline_failover_primitives.md) | Add runtime region-offline and write-region failover store primitives (PR2). | +| [ADR-008](adr/008_defer_auth_https.md) | Defer HTTPS and authentication to a dedicated later PR. | +| [ADR-009](adr/009_ci_reuse_existing_suites.md) | Validate via the existing suites over a new `emulator_inmemory` cfg, in both gateway modes. | +| [ADR-010](adr/010_pr_sequencing.md) | Sequence the work as PR1 (hosting + control plane + CI), PR2 (primitives), PR3 (auth/HTTPS). | + +--- + +## 12. Open Questions & Future Work + +- **YAML config.** Deferred; `serde_yaml` is unmaintained, so a maintained crate would be added + in a follow-up if demand warrants (ADR-005). +- **h2c fallback hardening.** If validation shows the driver does not cleanly fall back from h2c + to HTTP/1.1 against a cleartext HTTP/1.1-only server, broaden `has_explicit_http2_incompatibility` + (ADR-003). Not required while the host always serves h2c. +- **Additional control-plane ops.** Throttling toggles, forced session-not-available, and + replication-delay overrides could be surfaced through the management API later if cross-SDK + tests need them. + +--- + +## 13. References + +- In-memory emulator spec: `../../azure_data_cosmos_driver/docs/in-memory-emulator-spec.md` +- Gateway V2 spec: `../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` +- Transport pipeline spec: `../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` +- RNTBD codec: `../../azure_data_cosmos_driver/src/driver/transport/rntbd/` +- Cosmos test setup: `../../eng/scripts/Invoke-CosmosTestSetup.ps1` From 3ce86e85437ff79b1a625f2f9cc47c148736832b Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 12:56:30 +0000 Subject: [PATCH 02/44] Update plan.md --- .../azure_data_cosmos_emulator/docs/plan.md | 189 ++++++++++++------ 1 file changed, 127 insertions(+), 62 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 5fca0b00809..5ead41501d1 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -58,11 +58,11 @@ emulator implementation out of the driver crate (it depends heavily on driver-in The work ships across three pull requests. -| PR | Contents | Feature/cfg | -| --- | --- | --- | +| PR | Contents | Feature/cfg | +| ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | | **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway V2 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator_host`, `test_category = "emulator_inmemory"` | -| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | -| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | +| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | +| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | ### Feature gating @@ -78,17 +78,31 @@ The work ships across three pull requests. ## 3. Architecture Overview -```text - ┌────────────────────────────────────────────────┐ - │ azure_data_cosmos_emulator (bin) │ - │ │ - client SDK ──h2c──▶ │ Region "East US" gateway listener :8081 ─┐ │ - (any lang) │ Region "East US" thin-client :8444 ─┤ │ - │ Region "West US" gateway listener :8082 ─┼──▶ │ Arc - operator ──http─▶ │ Region "West US" thin-client :8445 ─┤ │ └── Arc - (curl) │ Management API :9090 ─┘ │ - └────────────────────────────────────────────────┘ - (all listeners share ONE store) +```mermaid +flowchart LR + SDK["Client SDK
(any language)"] + OP["Operator
(curl)"] + + subgraph HOST["azure_data_cosmos_emulator (binary)"] + direction TB + EG["East US gateway :8081"] + ET["East US thin-client :8444"] + WG["West US gateway :8082"] + WT["West US thin-client :8445"] + MGMT["Management API :9090"] + STORE["Single shared store
Arc<InMemoryEmulatorHttpClient> owns Arc<EmulatorStore>"] + EG --> STORE + ET --> STORE + WG --> STORE + WT --> STORE + MGMT --> STORE + end + + SDK -- h2c --> EG + SDK -- h2c --> ET + SDK -- h2c --> WG + SDK -- h2c --> WT + OP -- http --> MGMT ``` - **Port-per-region.** Each virtual region is a distinct `127.0.0.1:{port}` gateway endpoint. @@ -108,19 +122,20 @@ The work ships across three pull requests. ### Crate layout (proposed) -```text -sdk/cosmos/azure_data_cosmos_emulator/ -├── Cargo.toml # publish = false; deps: clap, axum, tokio, serde, serde_json, tracing -├── README.md -├── docs/ -│ ├── plan.md # this document -│ └── adr/ # architecture decision records -└── src/ - ├── main.rs # CLI (clap), startup, listener wiring - ├── config.rs # serde DTOs + translation to driver types + seeding - ├── data_plane.rs # HTTP <-> azure_core::http::Request bridge (Gateway V1) - ├── gateway_v2.rs # thin-client listener, connectivity probe, RNTBD bridge - └── management.rs # control-plane REST API (axum router) +```mermaid +flowchart TB + ROOT["azure_data_cosmos_emulator/"] + ROOT --> CARGO["Cargo.toml
publish = false; clap, axum, tokio, serde, serde_json, tracing"] + ROOT --> README["README.md"] + ROOT --> DOCS["docs/"] + ROOT --> SRC["src/"] + DOCS --> PLAN["plan.md: this document"] + DOCS --> ADR["adr/: architecture decision records"] + SRC --> MAIN["main.rs: CLI, startup, listener wiring"] + SRC --> CONFIG["config.rs: serde DTOs, translate to driver types, seeding"] + SRC --> DP["data_plane.rs: HTTP to azure_core Request bridge (Gateway V1)"] + SRC --> GW2["gateway_v2.rs: thin-client listener, connectivity probe, RNTBD bridge"] + SRC --> MGMT["management.rs: control-plane REST API (axum router)"] ``` --- @@ -173,22 +188,22 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-005. ### 4.2 Field reference -| Path | Type | Notes | -| --- | --- | --- | -| `account.writeMode` | `"single" \| "multi"` | Maps to `WriteMode`. In `single`, the first region is the hub/write region. | -| `account.consistency` | enum | `Strong \| BoundedStaleness \| Session \| ConsistentPrefix \| Eventual`. | -| `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | -| `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | -| `account.regions[].gatewayPort` | u16 | Gateway V1 (JSON REST) port. Region endpoint = `http://127.0.0.1:{gatewayPort}`. | -| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway V2 simulation for the region. | -| `account.regions[].regionId` | u64, optional | Auto-assigned by position when omitted. | -| `account.replication` | object | Default replication delay + buffer cap. | -| `account.replicationOverrides[]` | array | Per source→target replication overrides. | -| `management.port` | u16 | Control-plane REST API port. | -| `databases[].containers[].partitionKey` | object | Standard Cosmos partition key definition (`paths`, `kind`, `version`). | -| `databases[].containers[].partitionCount` | u32 | Initial physical partition count. | -| `databases[].containers[].throughput` | u32 | Provisioned RU/s (drives throttling when enabled). | -| `databases[].containers[].seedItems[]` | array | Documents created on startup; each carries its `partitionKey` value array and the `document` body. | +| Path | Type | Notes | +| ----------------------------------------- | --------------------- | -------------------------------------------------------------------------------------------------- | +| `account.writeMode` | `"single" \| "multi"` | Maps to `WriteMode`. In `single`, the first region is the hub/write region. | +| `account.consistency` | enum | `Strong \| BoundedStaleness \| Session \| ConsistentPrefix \| Eventual`. | +| `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | +| `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | +| `account.regions[].gatewayPort` | u16 | Gateway V1 (JSON REST) port. Region endpoint = `http://127.0.0.1:{gatewayPort}`. | +| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway V2 simulation for the region. | +| `account.regions[].regionId` | u64, optional | Auto-assigned by position when omitted. | +| `account.replication` | object | Default replication delay + buffer cap. | +| `account.replicationOverrides[]` | array | Per source→target replication overrides. | +| `management.port` | u16 | Control-plane REST API port. | +| `databases[].containers[].partitionKey` | object | Standard Cosmos partition key definition (`paths`, `kind`, `version`). | +| `databases[].containers[].partitionCount` | u32 | Initial physical partition count. | +| `databases[].containers[].throughput` | u32 | Provisioned RU/s (drives throttling when enabled). | +| `databases[].containers[].seedItems[]` | array | Documents created on startup; each carries its `partitionKey` value array and the `document` body. | Seed items are created through the same request path as real writes (a synthesized create-item request per item), so EPK routing, RU accounting, and replication behave identically to @@ -246,19 +261,58 @@ GET /account ### 6.2 Partition split / merge +A split accepts an optional JSON body that selects **how the split boundary is chosen**. Three +modes are supported, mirroring the real service and the emulator's existing split hooks: + +| `mode` | Split boundary | Backing store API | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| `midpoint` (default) | Geometric midpoint of the partition's EPK range, independent of the data distribution. | `split_partition` (existing) | +| `epk` | An explicit EPK boundary supplied as a hex string (same form as the `minInclusive` / `maxExclusive` values in the `/pkranges` response). | `split_partition_at_epk` (existing) + a hex-to-`Epk` parser (small addition) | +| `storage` | An EPK computed to balance document count / storage across the two children, the way the service splits under storage pressure. | `split_partition_by_storage` (new helper) | + ```text POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split - → 202 { "database": "testdb", "container": "testcoll", "parent": 0, "children": [4, 5] } + body (optional): { + "mode": "midpoint" | "epk" | "storage", // default: "midpoint" + "epk": "", // required when mode = "epk"; ignored otherwise + "lockDurationMs": 0 // optional; 410/1007 window before children appear + } + → 202 { + "database": "testdb", "container": "testcoll", + "parent": 0, "children": [4, 5], + "mode": "storage", "splitEpk": "6A3C000000000000000000000000000000" + } POST /databases/{db}/containers/{coll}/partitions/merge - body: { "partitionIds": [4, 5] } + body: { "partitionIds": [4, 5] } // exactly two adjacent partitions → 202 { "merged": [4, 5], "into": 6 } ``` -Sample: +The response always echoes the resolved `mode` and the concrete `splitEpk` that was applied, so a +caller that requested `midpoint` or `storage` learns the boundary the emulator chose. Only the +`epk` hex parser and the `storage` mode are new code behind this endpoint; `midpoint` and the +`epk` split execution reuse existing store hooks. + +Samples — one request per split mode: ```bash +# 1) Mid-point split (default): halve the partition's EPK range geometrically. curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split + +# 2) Custom-EPK split: split at an explicit hex EPK boundary. +curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ + -H 'content-type: application/json' \ + -d '{ "mode": "epk", "epk": "3FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" }' + +# 3) Storage-based split: pick a boundary that balances documents across the children. +curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ + -H 'content-type: application/json' \ + -d '{ "mode": "storage" }' + +# Any mode may set a non-zero lock window; the partition returns 410/1007 until it elapses. +curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ + -H 'content-type: application/json' \ + -d '{ "mode": "midpoint", "lockDurationMs": 500 }' ``` ### 6.3 Region offline / online (PR2) @@ -379,9 +433,20 @@ async fn serve_cosmos( ### 7.3 Control-plane primitives ```rust -// Already public today (behind the base emulator feature): -store.split_partition("testdb", "testcoll", 0); -store.merge_partitions("testdb", "testcoll", &[4, 5]); +use std::time::Duration; + +// Split — existing store hooks (behind the base emulator feature): +store.split_partition("testdb", "testcoll", 0, Duration::ZERO); // REST mode = "midpoint" +// `split_epk` is an Epk parsed from the request's hex boundary (REST mode = "epk"). +store.split_partition_at_epk("testdb", "testcoll", 0, split_epk, Duration::ZERO); + +// Split — storage-based (new helper): computes a balancing EPK from the live doc distribution. +store.split_partition_by_storage("testdb", "testcoll", 0, Duration::ZERO); // REST mode = "storage" + +// Merge exactly two adjacent partitions: +store.merge_partitions("testdb", "testcoll", 4, 5, Duration::ZERO); + +// Replication + per-partition failover (existing): store.pause_replication("West US"); store.resume_replication("West US"); store.set_per_partition_failover(true); @@ -456,18 +521,18 @@ hosting and control-plane work (ADR-008). ## 11. Architecture Decision Records -| ADR | Decision | -| --- | --- | -| [ADR-001](adr/001_separate_host_crate.md) | Host in a separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. | -| [ADR-002](adr/002_port_per_region.md) | Model each region as a distinct localhost port; one shared store. | -| [ADR-003](adr/003_cleartext_http2.md) | Serve cleartext HTTP/2 (h2c) and reuse the driver's existing prior-knowledge probe. | -| [ADR-004](adr/004_management_rest_api.md) | Expose emulator-only control-plane actions via a separate management REST API. | -| [ADR-005](adr/005_json_config_startup_seed.md) | Drive startup topology and seed data from a JSON config file; defer YAML. | -| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway V2 by promoting the test-only inverse RNTBD codec to production, config-gated. | -| [ADR-007](adr/007_region_offline_failover_primitives.md) | Add runtime region-offline and write-region failover store primitives (PR2). | -| [ADR-008](adr/008_defer_auth_https.md) | Defer HTTPS and authentication to a dedicated later PR. | -| [ADR-009](adr/009_ci_reuse_existing_suites.md) | Validate via the existing suites over a new `emulator_inmemory` cfg, in both gateway modes. | -| [ADR-010](adr/010_pr_sequencing.md) | Sequence the work as PR1 (hosting + control plane + CI), PR2 (primitives), PR3 (auth/HTTPS). | +| ADR | Decision | +| -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| [ADR-001](adr/001_separate_host_crate.md) | Host in a separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. | +| [ADR-002](adr/002_port_per_region.md) | Model each region as a distinct localhost port; one shared store. | +| [ADR-003](adr/003_cleartext_http2.md) | Serve cleartext HTTP/2 (h2c) and reuse the driver's existing prior-knowledge probe. | +| [ADR-004](adr/004_management_rest_api.md) | Expose emulator-only control-plane actions via a separate management REST API. | +| [ADR-005](adr/005_json_config_startup_seed.md) | Drive startup topology and seed data from a JSON config file; defer YAML. | +| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway V2 by promoting the test-only inverse RNTBD codec to production, config-gated. | +| [ADR-007](adr/007_region_offline_failover_primitives.md) | Add runtime region-offline and write-region failover store primitives (PR2). | +| [ADR-008](adr/008_defer_auth_https.md) | Defer HTTPS and authentication to a dedicated later PR. | +| [ADR-009](adr/009_ci_reuse_existing_suites.md) | Validate via the existing suites over a new `emulator_inmemory` cfg, in both gateway modes. | +| [ADR-010](adr/010_pr_sequencing.md) | Sequence the work as PR1 (hosting + control plane + CI), PR2 (primitives), PR3 (auth/HTTPS). | --- From ee5fc91ee74dfff24c806280c3a6e9aa06d85d91 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 13:00:40 +0000 Subject: [PATCH 03/44] Update plan.md --- .../azure_data_cosmos_emulator/docs/plan.md | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 5ead41501d1..e7ad5390b6b 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -122,20 +122,19 @@ flowchart LR ### Crate layout (proposed) -```mermaid -flowchart TB - ROOT["azure_data_cosmos_emulator/"] - ROOT --> CARGO["Cargo.toml
publish = false; clap, axum, tokio, serde, serde_json, tracing"] - ROOT --> README["README.md"] - ROOT --> DOCS["docs/"] - ROOT --> SRC["src/"] - DOCS --> PLAN["plan.md: this document"] - DOCS --> ADR["adr/: architecture decision records"] - SRC --> MAIN["main.rs: CLI, startup, listener wiring"] - SRC --> CONFIG["config.rs: serde DTOs, translate to driver types, seeding"] - SRC --> DP["data_plane.rs: HTTP to azure_core Request bridge (Gateway V1)"] - SRC --> GW2["gateway_v2.rs: thin-client listener, connectivity probe, RNTBD bridge"] - SRC --> MGMT["management.rs: control-plane REST API (axum router)"] +```text +sdk/cosmos/azure_data_cosmos_emulator/ +├── Cargo.toml # publish = false; deps: clap, axum, tokio, serde, serde_json, tracing +├── README.md +├── docs/ +│ ├── plan.md # this document +│ └── adr/ # architecture decision records +└── src/ + ├── main.rs # CLI (clap), startup, listener wiring + ├── config.rs # serde DTOs + translation to driver types + seeding + ├── data_plane.rs # HTTP <-> azure_core::http::Request bridge (Gateway V1) + ├── gateway_v2.rs # thin-client listener, connectivity probe, RNTBD bridge + └── management.rs # control-plane REST API (axum router) ``` --- From 77205634151bd0009722c881bb0aca22058ce85d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:32:44 +0200 Subject: [PATCH 04/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md index 6902962a43e..24cf51cf3bc 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md @@ -13,7 +13,7 @@ reviewable increments that each land something coherent. Sequence the work as three PRs: -- **PR1** — the new host crate; per-region h2c hosting of Gateway V1 **and** Gateway V2 +- **PR1** — the new host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); the management REST API over the control-plane actions that map to existing store methods; and CI running the existing suites against the hosted emulator in both gateway modes. - **PR2** — the new store primitives (region offline/online, runtime write-region failover) with From 9dc8ec403652d6b7a401f105c2671ce4e96a7abd Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:33:12 +0200 Subject: [PATCH 05/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/002_port_per_region.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md index 519673e81d2..43544d9ac3c 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md @@ -13,8 +13,8 @@ already resolves a request's region by `(scheme, host, port)`. Bind one gateway listener per region on a distinct `127.0.0.1:{port}`. A single shared `EmulatorStore` backs every listener; the region is resolved per request from the `Host` header. -A single-region account is simply one port. Gateway V2 adds an optional second (thin-client) port -per region. +A single-region account is simply one port. Gateway 2.0 adds an optional second RNTBD port per +region. ## Consequences From 8863aa977e909f90c247fae090e0772c11b4f255 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:33:41 +0200 Subject: [PATCH 06/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/003_cleartext_http2.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md index 5e09851a6cf..b272e7b798f 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md @@ -11,11 +11,12 @@ negotiate h2c. ## Decision -Serve h2c from the host using `axum::serve` (hyper-util's `auto` builder accepts HTTP/2 -prior-knowledge connections automatically) and rely on the driver's **existing** behavior: -the `Http2Only` reqwest client already sets `http2_prior_knowledge()` (h2c against `http://`), -initialization already probes HTTP/2 then falls back to HTTP/1.1, and `http://` is already -permitted for loopback emulator hosts. No mandatory driver change. +Serve h2c from the host using `axum::serve`, explicitly enabling axum's non-default `http2` +feature so hyper-util's `auto` builder accepts HTTP/2 prior-knowledge connections. Then rely on the +driver's **existing** behavior: the `Http2Only` reqwest client already sets +`http2_prior_knowledge()` (h2c against `http://`), initialization already probes HTTP/2 then falls +back to HTTP/1.1, and `http://` is already permitted for loopback emulator hosts. No mandatory +driver change. ## Consequences From 3c2b1a328092642afa75e6b7a27c55d3025e9812 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:34:14 +0200 Subject: [PATCH 07/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/006_gateway_v2_rntbd.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md index 6427791ebb3..fa285d4e4de 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md @@ -8,9 +8,10 @@ Gateway V2 (thin-client) uses the RNTBD wire format over HTTP/2 instead of JSON REST. To let the hosted emulator exercise the driver's Gateway V2 path, the emulator must act as an RNTBD **server**: answer the connectivity probe, decode inbound request frames, and encode outbound response frames. -The driver already owns the **client** halves (`RntbdRequestFrame::write`, `RntbdResponse::read`); -the inverse halves exist only as `#[cfg(test)]` helpers. Today the emulator advertises no -thin-client endpoints, so the driver always suppresses Gateway V2 against it. +The driver already owns the **client** halves (`RntbdRequestFrame::write`, `RntbdResponse::read`). +Tests contain reusable pieces of the inverse logic—a test-local request parser and direct response +frame construction—but no inverse associated methods yet. Today the emulator advertises no +Gateway 2.0 endpoints, so the driver always suppresses Gateway 2.0 against it. ## Decision From ca92a9f305b5a6721b80d794e2c4f279fc986f19 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:34:34 +0200 Subject: [PATCH 08/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/009_ci_reuse_existing_suites.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md index 13fe0295659..bd5b6bd4f17 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md @@ -12,9 +12,11 @@ against it. The suites are gated at runtime by a `test_category` cfg set via `RU ## Decision Add a new `test_category = "emulator_inmemory"` cfg (registered in the `build.rs` of both -`azure_data_cosmos` and `azure_data_cosmos_driver`). Extend +`azure_data_cosmos` and `azure_data_cosmos_driver`) and add it to the existing emulator suites' +`#[cfg_attr(...)]` gates and ignore messages, preserving intentional legacy-only exclusions. Extend `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary with a provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the +hosted endpoint. hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on `Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway V2 (thin-client) modes. This CI pass replaces a standalone HTTP/2 validation spike. From 6795fe1056df4847f08f1f125b816975ca090461 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:36:05 +0200 Subject: [PATCH 09/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/009_ci_reuse_existing_suites.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md index bd5b6bd4f17..7b5ade8b80b 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md @@ -18,8 +18,8 @@ Add a new `test_category = "emulator_inmemory"` cfg (registered in the `build.rs provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the hosted endpoint. hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on -`Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway V2 -(thin-client) modes. This CI pass replaces a standalone HTTP/2 validation spike. +`Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway 2.0 +(RNTBD) modes. This CI pass replaces a standalone HTTP/2 validation spike. ## Consequences From 9fcdda975f58cf55d1f491fb3af37fa6011794f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:39:31 +0000 Subject: [PATCH 10/44] docs: require HTTPS for auth modes and define JWT trust inputs in ADR-008 Address reviewer feedback: key/entra auth must require HTTPS (else credentials leak over plaintext h2c), and entra mode must define the full set of JWT validation trust inputs (JWKS source, issuer, audience, allowed OIDs) rather than rely on an allow-list alone. Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../docs/adr/008_defer_auth_https.md | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md index ee70baa53ad..4b6a5997377 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md @@ -16,13 +16,33 @@ behind an `auth` config block and CLI flags: - Optional HTTPS via axum + rustls (`--https --cert --key`); h2c remains the default when off. - Auth modes: `none` (default), `key` (validate the `Authorization` HMAC against a primary key and - a primary read-only key, matching the service), and `entra` (validate a bearer JWT and check its - object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). + a primary read-only key, matching the service), and `entra` (validate a bearer JWT; see trust + inputs below). +- **HTTPS is required when `key` or `entra` auth is selected.** Enabling either auth mode without + `--https` is a startup error; the process exits with a descriptive message. This prevents account + keys or bearer tokens from being transmitted over plaintext h2c. + +### Entra JWT trust inputs + +An allow-list of object IDs / app IDs is _not_ sufficient to authenticate a JWT. PR3 must define +and validate the following trust inputs before the `entra` mode can be accepted: + +| Input | Flag / config key | Purpose | +|---|---|---| +| JWKS source | `--jwks-uri` or `--jwks-file` | Provides the public keys used to verify the JWT signature. The offline goal recommends `--jwks-file`; `--jwks-uri` is used for online (AAD-connected) deployments. | +| Expected issuer | `--issuer` | The `iss` claim value that the emulator will accept; prevents tokens issued by a different authority from being accepted. | +| Expected audience | `--audience` | The `aud` claim value, typically the emulator's resource URI; prevents tokens issued for other apps from being replayed. | +| Allowed OIDs / app IDs | `--allowed-oid` (repeatable) | After the signature, issuer, and audience are validated, restrict access to the listed object or application IDs. | + +All four inputs must be provided when `entra` mode is enabled; the process exits with a descriptive +error if any are missing. ## Consequences The initial hosting and control-plane work ships without certificate or token friction, and can be -validated over h2c immediately. Auth and HTTPS land as a self-contained, reviewable increment. +validated over h2c immediately. Auth and HTTPS land as a self-contained, reviewable increment, with +a security-sound design: authenticated modes are gated behind HTTPS, and JWT validation is complete +rather than rely on an allow-list alone. ## Alternatives From 67bc248e86db684db36f11aade846ee57c1c277f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:38:50 +0000 Subject: [PATCH 11/44] Use Gateway 2.0 naming Normalize ADR-003 to the repository's canonical Gateway 2.0 terminology. This keeps the document aligned with the Cosmos Gateway naming guidance without changing any other ADR content. Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../docs/adr/003_cleartext_http2.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md index b272e7b798f..326382741b7 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md @@ -5,7 +5,7 @@ ## Context -HTTP/2 is a hard requirement for Gateway V2. PR1 hosts plaintext HTTP (TLS is deferred), so the +HTTP/2 is a hard requirement for Gateway 2.0. PR1 hosts plaintext HTTP (TLS is deferred), so the data plane must run cleartext HTTP/2 (h2c). The question was whether a driver change is needed to negotiate h2c. @@ -28,7 +28,7 @@ cleartext HTTP/1.1-only server. ## Alternatives - Serving HTTP/1.1 in PR1 and deferring HTTP/2 to the TLS PR was rejected: it would leave the - Gateway V2 path without validation for longer and understate the HTTP/2 requirement. + Gateway 2.0 path without validation for longer and understate the HTTP/2 requirement. - Adding a new client toggle to force prior knowledge was rejected as redundant with existing behavior. From 80b550eeb639a3484a5e43b960089bf29b563509 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:39:52 +0000 Subject: [PATCH 12/44] docs: normalize Gateway 2.0 naming Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../docs/adr/006_gateway_v2_rntbd.md | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md index fa285d4e4de..41f1112ad23 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md @@ -1,12 +1,12 @@ -# ADR-006 — Simulate Gateway V2 by promoting the inverse RNTBD codec, config-gated +# ADR-006 — Simulate Gateway 2.0 by promoting the inverse RNTBD codec, config-gated **Status:** Accepted **Date:** 2026-07-14 ## Context -Gateway V2 (thin-client) uses the RNTBD wire format over HTTP/2 instead of JSON REST. To let the -hosted emulator exercise the driver's Gateway V2 path, the emulator must act as an RNTBD **server**: +Gateway 2.0 uses the RNTBD wire format over HTTP/2 instead of JSON REST. To let the hosted +emulator exercise the driver's Gateway 2.0 path, the emulator must act as an RNTBD **server**: answer the connectivity probe, decode inbound request frames, and encode outbound response frames. The driver already owns the **client** halves (`RntbdRequestFrame::write`, `RntbdResponse::read`). Tests contain reusable pieces of the inverse logic—a test-local request parser and direct response @@ -15,30 +15,31 @@ Gateway 2.0 endpoints, so the driver always suppresses Gateway 2.0 against it. ## Decision -Fold Gateway V2 simulation into PR1, gated by config. Promote the test-only inverse codec +Fold Gateway 2.0 simulation into PR1, gated by config. Promote the test-only inverse codec (`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, **co-located** in `src/driver/transport/rntbd/` behind the host feature, reusing the existing token and status -primitives. When a region declares a `thinClientPort`, the emulator binds a thin-client listener -that answers `POST /connectivity-probe` with `200`, decodes RNTBD + thin-client headers into a -logical operation, dispatches through the shared store, and encodes an RNTBD response; the account -topology advertises `thinClient{Readable,Writable}Locations`. +primitives. When a region declares a `thinClientPort`, the emulator binds a Gateway 2.0 listener +that answers `POST /connectivity-probe` with `200`, decodes RNTBD + `thinclient` wire headers into +a logical operation, dispatches through the shared store, and encodes an RNTBD response; the +account topology advertises `thinClient{Readable,Writable}Locations`. ## Consequences -Gateway V1 always works; Gateway V2 is opt-in per region, so CI can run the existing suites in both -modes. Because the request pipeline selects Gateway V2 purely from thin-client advertisement, no -driver routing change is needed. The codec halves stay symmetric and co-located for maximal reuse. +Gateway V1 always works; Gateway 2.0 is opt-in per region, so CI can run the existing suites in +both modes. Because the request pipeline selects Gateway 2.0 purely from the +`thinClient{Readable,Writable}Locations` advertisement, no driver routing change is needed. The +codec halves stay symmetric and co-located for maximal reuse. ## Alternatives -- Shipping Gateway V2 as a separate later PR was rejected: hosting both from the start lets one CI - pass validate both protocol paths end-to-end. +- Shipping Gateway 2.0 as a separate later PR was rejected: hosting both from the start lets one + CI pass validate both protocol paths end-to-end. - Placing the server-side codec in the emulator module was rejected in favor of co-location with the client codec (Option A), which maximizes reuse and keeps the wire format in one place. -- Always advertising thin-client endpoints was rejected: config-gating keeps Gateway V1 as the +- Always advertising Gateway 2.0 endpoints was rejected: config-gating keeps Gateway V1 as the safe default. ## References - Plan & summary: ../plan.md -- Gateway V2 spec: ../../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md +- Gateway 2.0 spec: ../../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md From 2399f360ba747dedb9efd11d1952218993d0842c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:41:52 +0000 Subject: [PATCH 13/44] Adjust split/merge plan contract Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../azure_data_cosmos_emulator/docs/plan.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index e7ad5390b6b..0ab77175cf9 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -276,21 +276,23 @@ POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split "epk": "", // required when mode = "epk"; ignored otherwise "lockDurationMs": 0 // optional; 410/1007 window before children appear } - → 202 { - "database": "testdb", "container": "testcoll", - "parent": 0, "children": [4, 5], - "mode": "storage", "splitEpk": "6A3C000000000000000000000000000000" - } + → 202 { "operationId": "op-split-123", "status": "Running" } POST /databases/{db}/containers/{coll}/partitions/merge body: { "partitionIds": [4, 5] } // exactly two adjacent partitions - → 202 { "merged": [4, 5], "into": 6 } + → 202 { "operationId": "op-merge-456", "status": "Running" } + +GET /operations/{operationId} + → 200 { "operationId": "...", "status": "Running" | "Succeeded" | "Failed" } + // On success, include operation-specific result details: + // split: { "database": "testdb", "container": "testcoll", "parent": 0, "children": [4, 5], "mode": "storage", "splitEpk": "6A3C000000000000000000000000000000" } + // merge: { "merged": [4, 5], "into": 6 } ``` -The response always echoes the resolved `mode` and the concrete `splitEpk` that was applied, so a -caller that requested `midpoint` or `storage` learns the boundary the emulator chose. Only the -`epk` hex parser and the `storage` mode are new code behind this endpoint; `midpoint` and the -`epk` split execution reuse existing store hooks. +The operation result (via `GET /operations/{operationId}`) echoes the resolved `mode` and the +concrete `splitEpk` that was applied, so a caller that requested `midpoint` or `storage` learns +the boundary the emulator chose. Only the `epk` hex parser and the `storage` mode are new code +behind this endpoint; `midpoint` and the `epk` split execution reuse existing store hooks. Samples — one request per split mode: From bf4c5df6e1710b90fc10f7af43c1c86927be19a2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:43:00 +0000 Subject: [PATCH 14/44] Normalize Gateway 2.0 plan naming Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../azure_data_cosmos_emulator/docs/plan.md | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 0ab77175cf9..165c1c474a9 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -13,7 +13,7 @@ 2. [Scope, Phasing & Feature Gating](#2-scope-phasing--feature-gating) 3. [Architecture Overview](#3-architecture-overview) 4. [Configuration File](#4-configuration-file) -5. [Data-Plane Hosting (Gateway V1 & Gateway V2)](#5-data-plane-hosting-gateway-v1--gateway-v2) +5. [Data-Plane Hosting (Gateway V1 & Gateway 2.0)](#5-data-plane-hosting-gateway-v1--gateway-20) 6. [Management REST API (Control Plane)](#6-management-rest-api-control-plane) 7. [Rust Public API Surface](#7-rust-public-api-surface) 8. [HTTP/2 & Transport Notes](#8-http2--transport-notes) @@ -43,7 +43,7 @@ emulator implementation out of the driver crate (it depends heavily on driver-in just enough **public** surface — gated behind a feature flag the host crate sets automatically — to construct, seed, control, and serve the emulator. - **Wire fidelity.** The hosted emulator speaks the real Cosmos DB wire protocol: Gateway V1 - (JSON REST) and, when enabled, Gateway V2 (thin-client / RNTBD over HTTP/2). Existing SDK test + (JSON REST) and, when enabled, Gateway 2.0 (RNTBD over HTTP/2). Existing SDK test suites run against it unchanged. - **Deterministic, offline, no Docker.** No network access, no external accounts, no containers required to run the store itself. @@ -58,11 +58,11 @@ emulator implementation out of the driver crate (it depends heavily on driver-in The work ships across three pull requests. -| PR | Contents | Feature/cfg | -| ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | -| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway V2 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator_host`, `test_category = "emulator_inmemory"` | -| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | -| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | +| PR | Contents | Feature/cfg | +| ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator_host`, `test_category = "emulator_inmemory"` | +| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | +| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | ### Feature gating @@ -86,9 +86,9 @@ flowchart LR subgraph HOST["azure_data_cosmos_emulator (binary)"] direction TB EG["East US gateway :8081"] - ET["East US thin-client :8444"] + ET["East US Gateway 2.0 :8444"] WG["West US gateway :8082"] - WT["West US thin-client :8445"] + WT["West US Gateway 2.0 :8445"] MGMT["Management API :9090"] STORE["Single shared store
Arc<InMemoryEmulatorHttpClient> owns Arc<EmulatorStore>"] EG --> STORE @@ -114,8 +114,8 @@ flowchart LR - **Data-plane bridge.** Each gateway listener rebuilds an `azure_core::http::Request` from the incoming HTTP request and calls `InMemoryEmulatorHttpClient::execute_request`, then converts the `AsyncRawResponse` back to an HTTP response. -- **Gateway V2 (optional).** When a region declares a `thinClientPort`, the host binds a - thin-client listener that answers the connectivity probe and speaks RNTBD; the store's account +- **Gateway 2.0 (optional).** When a region declares a `thinClientPort`, the host binds a + Gateway 2.0 listener that answers the connectivity probe and speaks RNTBD; the store's account topology then advertises `thinClient{Readable,Writable}Locations`. - **Management port.** A dedicated port serves the control-plane REST API, kept separate from the Cosmos wire protocol so the two never collide. @@ -133,7 +133,7 @@ sdk/cosmos/azure_data_cosmos_emulator/ ├── main.rs # CLI (clap), startup, listener wiring ├── config.rs # serde DTOs + translation to driver types + seeding ├── data_plane.rs # HTTP <-> azure_core::http::Request bridge (Gateway V1) - ├── gateway_v2.rs # thin-client listener, connectivity probe, RNTBD bridge + ├── gateway_v2.rs # Gateway 2.0 listener, connectivity probe, RNTBD bridge └── management.rs # control-plane REST API (axum router) ``` @@ -194,7 +194,7 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-005. | `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | | `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | | `account.regions[].gatewayPort` | u16 | Gateway V1 (JSON REST) port. Region endpoint = `http://127.0.0.1:{gatewayPort}`. | -| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway V2 simulation for the region. | +| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway 2.0 simulation for the region. | | `account.regions[].regionId` | u64, optional | Auto-assigned by position when omitted. | | `account.replication` | object | Default replication delay + buffer cap. | | `account.replicationOverrides[]` | array | Per source→target replication overrides. | @@ -210,7 +210,7 @@ client-issued writes. --- -## 5. Data-Plane Hosting (Gateway V1 & Gateway V2) +## 5. Data-Plane Hosting (Gateway V1 & Gateway 2.0) ### 5.1 Gateway V1 (JSON REST) @@ -219,22 +219,22 @@ delegates to `execute_request`. All existing point operations and Cosmos control operations that are part of the gateway contract (database/container/offer CRUD, PK-ranges, account read) are served here unchanged. -### 5.2 Gateway V2 (thin-client / RNTBD) +### 5.2 Gateway 2.0 (RNTBD over HTTP/2) Enabled per region by setting `thinClientPort`. When enabled: 1. The account topology advertises `thinClientReadableLocations` / `thinClientWritableLocations` pointing at `http://127.0.0.1:{thinClientPort}`. -2. The thin-client listener answers `POST /connectivity-probe` with `200 OK`. +2. The Gateway 2.0 listener answers `POST /connectivity-probe` with `200 OK`. 3. Data-plane requests arrive as RNTBD frames wrapped in HTTP/2 POSTs. The listener **decodes** - the RNTBD request frame + thin-client headers, reconstructs the logical operation, dispatches + the RNTBD request frame + literal `thinclient` headers, reconstructs the logical operation, dispatches it through the same store, and **encodes** the result as an RNTBD response frame. The driver already owns the client-side RNTBD codec (`RntbdRequestFrame::write`, `RntbdResponse::read`). This work promotes the currently test-only inverse halves (`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, co-located in `src/driver/transport/rntbd/` behind the host feature (ADR-006). Because the driver's request -pipeline decides Gateway V2 purely from whether the endpoint advertises a thin-client URL, +pipeline decides Gateway 2.0 purely from whether the endpoint advertises a Gateway 2.0 URL, advertising it plus serving RNTBD is sufficient — no driver routing change is required. --- @@ -458,7 +458,7 @@ store.set_region_online("West US"); store.set_write_region("West US")?; // runtime single-write failover; Err on unknown region ``` -### 7.4 Gateway V2 server codec (behind host feature) +### 7.4 Gateway 2.0 server codec (behind host feature) ```rust // Promoted from test-only helpers to production, co-located with the client codec. @@ -474,7 +474,7 @@ response.write(&mut out)?; // encode outbound (ser ## 8. HTTP/2 & Transport Notes -HTTP/2 is a hard requirement for Gateway V2. The relevant driver behavior **already exists**: +HTTP/2 is a hard requirement for Gateway 2.0. The relevant driver behavior **already exists**: - The `Http2Only` reqwest client sets `http2_prior_knowledge()`, which performs cleartext h2c against `http://` endpoints. @@ -500,7 +500,7 @@ incompatibility matcher is a **contingency** only if end-to-end validation revea to the hosted endpoint and the `emulator_inmemory` cfg. - Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` modeled on `Cosmos_vnext_emulator`, running the **existing** emulator suites against the hosted emulator in **both** Gateway V1 and - Gateway V2 (thin-client) modes. This is the real end-to-end validation of h2c + RNTBD. + Gateway 2.0 modes. This is the real end-to-end validation of h2c + RNTBD. --- @@ -529,7 +529,7 @@ hosting and control-plane work (ADR-008). | [ADR-003](adr/003_cleartext_http2.md) | Serve cleartext HTTP/2 (h2c) and reuse the driver's existing prior-knowledge probe. | | [ADR-004](adr/004_management_rest_api.md) | Expose emulator-only control-plane actions via a separate management REST API. | | [ADR-005](adr/005_json_config_startup_seed.md) | Drive startup topology and seed data from a JSON config file; defer YAML. | -| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway V2 by promoting the test-only inverse RNTBD codec to production, config-gated. | +| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway 2.0 by promoting the test-only inverse RNTBD codec to production, config-gated. | | [ADR-007](adr/007_region_offline_failover_primitives.md) | Add runtime region-offline and write-region failover store primitives (PR2). | | [ADR-008](adr/008_defer_auth_https.md) | Defer HTTPS and authentication to a dedicated later PR. | | [ADR-009](adr/009_ci_reuse_existing_suites.md) | Validate via the existing suites over a new `emulator_inmemory` cfg, in both gateway modes. | @@ -553,7 +553,7 @@ hosting and control-plane work (ADR-008). ## 13. References - In-memory emulator spec: `../../azure_data_cosmos_driver/docs/in-memory-emulator-spec.md` -- Gateway V2 spec: `../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` +- Gateway 2.0 spec: `../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` - Transport pipeline spec: `../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` - RNTBD codec: `../../azure_data_cosmos_driver/src/driver/transport/rntbd/` - Cosmos test setup: `../../eng/scripts/Invoke-CosmosTestSetup.ps1` From 909c6cd8b85efb8d768bff78c9954fa0e23732ce Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:45:46 +0200 Subject: [PATCH 15/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 165c1c474a9..5e304b47771 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -461,7 +461,7 @@ store.set_write_region("West US")?; // runtime single-write failover ### 7.4 Gateway 2.0 server codec (behind host feature) ```rust -// Promoted from test-only helpers to production, co-located with the client codec. +// Server-side codec methods extracted and completed from the existing test parsing/building logic. use azure_data_cosmos_driver::driver::transport::rntbd::{RntbdRequestFrame, RntbdResponse}; let frame = RntbdRequestFrame::read(&request_bytes)?; // decode inbound (server side) From e15a0024878faac86ef69263e7575c108cb24b4f Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:46:37 +0200 Subject: [PATCH 16/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 5e304b47771..b15f3df2318 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -495,6 +495,8 @@ incompatibility matcher is a **contingency** only if end-to-end validation revea - Register a new `test_category = "emulator_inmemory"` cfg in the `build.rs` of both `azure_data_cosmos` and `azure_data_cosmos_driver`. +- Add `test_category = "emulator_inmemory"` to the `#[cfg_attr(...)]` gates and ignore messages + of the existing emulator suites in both crates, preserving intentional legacy-only exclusions. - Extend `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary with a provisioning config, wait for `GET /health`, then set `AZURE_COSMOS_CONNECTION_STRING` to the hosted endpoint and the `emulator_inmemory` cfg. From b81eb51b4167c89d009914a20f9cc78cb10108bf Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:47:14 +0200 Subject: [PATCH 17/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index b15f3df2318..149c306d67d 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -483,11 +483,11 @@ HTTP/2 is a hard requirement for Gateway 2.0. The relevant driver behavior **alr - `ensure_endpoint_scheme_allowed` already permits `http://` for `localhost` / `127.0.0.1` / `[::1]` (and `AZURE_COSMOS_EMULATOR_HOST`). -`axum::serve` uses hyper-util's protocol-detecting `auto` builder, which accepts h2c -prior-knowledge connections automatically. So the host serves h2c, the existing probe negotiates -HTTP/2 against it, and no mandatory driver change is required. A targeted change to the -incompatibility matcher is a **contingency** only if end-to-end validation reveals a gap -(ADR-003). +`axum::serve` uses hyper-util's protocol-detecting `auto` builder and accepts h2c +prior-knowledge connections when axum is compiled with its non-default `http2` feature. The host +must enable that feature explicitly. With it enabled, the existing probe negotiates HTTP/2 against +the host and no mandatory driver change is required. A targeted change to the incompatibility +matcher is a **contingency** only if end-to-end validation reveals a gap (ADR-003). --- From e006e991f2fe5fc778af44335efeb6aa617a21fa Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:52:56 +0200 Subject: [PATCH 18/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../docs/adr/009_ci_reuse_existing_suites.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md index 7b5ade8b80b..f0052fa9d50 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md @@ -15,9 +15,7 @@ Add a new `test_category = "emulator_inmemory"` cfg (registered in the `build.rs `azure_data_cosmos` and `azure_data_cosmos_driver`) and add it to the existing emulator suites' `#[cfg_attr(...)]` gates and ignore messages, preserving intentional legacy-only exclusions. Extend `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary with a -provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the -hosted endpoint. -hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on +provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on `Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway 2.0 (RNTBD) modes. This CI pass replaces a standalone HTTP/2 validation spike. From 67e5fb27f8e37bb0faa147037ecab8c6ae170bca Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:53:19 +0200 Subject: [PATCH 19/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 149c306d67d..1d2e95ecef6 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -554,7 +554,7 @@ hosting and control-plane work (ADR-008). ## 13. References -- In-memory emulator spec: `../../azure_data_cosmos_driver/docs/in-memory-emulator-spec.md` +- In-memory emulator spec: `../../azure_data_cosmos/docs/in-memory-emulator-spec.md` - Gateway 2.0 spec: `../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` - Transport pipeline spec: `../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` - RNTBD codec: `../../azure_data_cosmos_driver/src/driver/transport/rntbd/` From 2ecd0e0f94d39bcb48d382dd3b1ef614f110c1a0 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:53:35 +0200 Subject: [PATCH 20/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md index 24cf51cf3bc..c4bb4607edc 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md @@ -31,7 +31,7 @@ paths. - Splitting hosting and CI into separate PRs was rejected: hosting without the CI validation proves nothing, so they ship together. -- Deferring Gateway V2 to its own PR was rejected: hosting both gateways from the start lets one CI +- Deferring Gateway 2.0 to its own PR was rejected: hosting both gateways from the start lets one CI pass cover both (ADR-006). ## References From 94af359cda670c0b9f59d23b5fcc0b602199793d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 17:53:53 +0200 Subject: [PATCH 21/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md index c4bb4607edc..152907aa049 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md @@ -24,7 +24,7 @@ Sequence the work as three PRs: PR1 is independently meaningful because the CI leg validates it end-to-end; it does not depend on the new primitives, which the existing suites do not exercise. PR2 and PR3 are self-contained, -testable increments. Gateway V2 is folded into PR1 so a single CI pass validates both protocol +testable increments. Gateway 2.0 is folded into PR1 so a single CI pass validates both protocol paths. ## Alternatives From 2dd214a75bed12751f0190c7b641d69ea595b227 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 16:03:09 +0000 Subject: [PATCH 22/44] Updating ADRs --- .../docs/adr/003_cleartext_http2.md | 39 +++++++------ .../docs/adr/004_management_rest_api.md | 4 +- .../docs/adr/005_json_config_startup_seed.md | 14 +++-- .../docs/adr/006_gateway_v2_rntbd.md | 40 ++++++------- .../docs/adr/007_dynamic_account_topology.md | 46 +++++++++++++++ .../007_region_offline_failover_primitives.md | 41 -------------- .../docs/adr/008_defer_auth_https.md | 56 ------------------- ...8_transport_security_and_authentication.md | 55 ++++++++++++++++++ .../docs/adr/009_ci_reuse_existing_suites.md | 39 ------------- .../docs/adr/010_pr_sequencing.md | 39 ------------- .../azure_data_cosmos_emulator/docs/plan.md | 33 ++++++----- 11 files changed, 174 insertions(+), 232 deletions(-) create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md delete mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md delete mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md delete mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md delete mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md index 326382741b7..97804071303 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md @@ -1,36 +1,39 @@ -# ADR-003 — Serve cleartext HTTP/2 (h2c) and reuse the existing probe +# ADR-003 — Support cleartext HTTP/2 for local emulator endpoints **Status:** Accepted **Date:** 2026-07-14 ## Context -HTTP/2 is a hard requirement for Gateway 2.0. PR1 hosts plaintext HTTP (TLS is deferred), so the -data plane must run cleartext HTTP/2 (h2c). The question was whether a driver change is needed to -negotiate h2c. +Gateway 2.0 requires HTTP/2. Local emulator scenarios also need to avoid certificate provisioning +and trust-store configuration when transport security is not under test. Cleartext HTTP/2 (h2c) +provides HTTP/2 framing on loopback without TLS, while production Cosmos endpoints remain HTTPS +only. ## Decision -Serve h2c from the host using `axum::serve`, explicitly enabling axum's non-default `http2` -feature so hyper-util's `auto` builder accepts HTTP/2 prior-knowledge connections. Then rely on the -driver's **existing** behavior: the `Http2Only` reqwest client already sets -`http2_prior_knowledge()` (h2c against `http://`), initialization already probes HTTP/2 then falls -back to HTTP/1.1, and `http://` is already permitted for loopback emulator hosts. No mandatory -driver change. +The host accepts h2c prior-knowledge connections on configured loopback endpoints. The Gateway 2.0 +listener rejects HTTP/1.x requests. The standard gateway listener may accept either HTTP/1.1 or +HTTP/2 so the driver's normal negotiation and fallback behavior remains observable. + +The driver permits `http://` Gateway 2.0 URLs only when the endpoint is recognized as an emulator +host. All other Gateway 2.0 endpoints must use `https://`. The existing `Http2Only` transport uses +`http2_prior_knowledge()` and the existing account probe determines whether standard gateway +traffic uses HTTP/2 or falls back to HTTP/1.1. ## Consequences -The hosted emulator negotiates HTTP/2 with the unmodified driver. A change to -`has_explicit_http2_incompatibility` is held as a **contingency**, applied only if end-to-end -validation reveals the driver does not cleanly fall back from h2c to HTTP/1.1 against a -cleartext HTTP/1.1-only server. +Gateway 2.0 can be exercised locally without certificates. Plaintext traffic is limited to +explicit emulator hosts, preserving the production transport boundary. Authentication modes that +carry credentials require TLS, as described by ADR-008. ## Alternatives -- Serving HTTP/1.1 in PR1 and deferring HTTP/2 to the TLS PR was rejected: it would leave the - Gateway 2.0 path without validation for longer and understate the HTTP/2 requirement. -- Adding a new client toggle to force prior knowledge was rejected as redundant with existing - behavior. +- Supporting only HTTP/1.1 was rejected because it cannot model Gateway 2.0. +- Requiring TLS for every local run was rejected because certificate setup would obscure tests + unrelated to transport security. +- Adding a client option to force prior knowledge was rejected because the existing HTTP/2-only + transport already provides that behavior. ## References diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md index 07fcb2f1221..3a38c54278d 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md @@ -8,8 +8,8 @@ Some control-plane actions are part of the Cosmos gateway contract (database/container/offer/item CRUD, PK-ranges, account read) and are already served on the region gateway ports. Others — partition split/merge, region offline/online, runtime write-region failover, per-partition -failover toggle, replication pause/resume — have **no** gateway equivalent and today exist only as -`EmulatorStore` method calls reachable in-process. +failover toggle, replication pause/resume — have **no** gateway equivalent. They require an +emulator-specific control surface when the emulator runs out of process. ## Decision diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md index 5619de34e31..c082bcb3266 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md @@ -16,21 +16,25 @@ Accept a single JSON file via `--config`. The host owns `serde` DTOs that mirror translate them into driver types (`VirtualAccountConfig`, `VirtualRegion`, `ContainerConfig`). Seed documents are created through the normal write path — one synthesized create-item request per item through `execute_request` — so EPK routing, RU accounting, and replication match -client-issued writes. The management REST API can further modify state at runtime. YAML is -deferred. +client-issued writes. The management REST API can further modify state at runtime. + +JSON is the canonical configuration representation. Additional syntaxes may be introduced only as +parsers that map to the same host-owned configuration model; they must not create a second set of +configuration semantics. ## Consequences Startup provisioning is declarative and reproducible; runtime mutation stays available through the control-plane API. Keeping the DTOs in the host crate leaves the driver's config types untouched. -JSON is supported first because `serde_json` is already a workspace dependency. +The configuration contract remains independent from the driver's internal structs and from any +particular parser implementation. ## Alternatives - Adding `serde` derives directly to the driver config types was rejected: several fields are not serializable, and it would leak host concerns into the driver. -- Shipping YAML in the first PR was rejected: `serde_yaml` is unmaintained; a maintained crate can - be added later if needed. +- Making YAML a second canonical contract was rejected because two independently evolving + representations would make validation and automation ambiguous. - Seeding items by writing store internals directly was rejected: routing through `execute_request` guarantees identical semantics to real writes. diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md index 41f1112ad23..4a09fec5671 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md @@ -1,4 +1,4 @@ -# ADR-006 — Simulate Gateway 2.0 by promoting the inverse RNTBD codec, config-gated +# ADR-006 — Keep hosted Gateway 2.0 framing in the driver **Status:** Accepted **Date:** 2026-07-14 @@ -9,33 +9,35 @@ Gateway 2.0 uses the RNTBD wire format over HTTP/2 instead of JSON REST. To let emulator exercise the driver's Gateway 2.0 path, the emulator must act as an RNTBD **server**: answer the connectivity probe, decode inbound request frames, and encode outbound response frames. The driver already owns the **client** halves (`RntbdRequestFrame::write`, `RntbdResponse::read`). -Tests contain reusable pieces of the inverse logic—a test-local request parser and direct response -frame construction—but no inverse associated methods yet. Today the emulator advertises no -Gateway 2.0 endpoints, so the driver always suppresses Gateway 2.0 against it. +The inverse codec must use the same token IDs, operation mappings, and framing rules as the client +codec. Duplicating those rules in the host crate would create two protocol implementations that +can drift. ## Decision -Fold Gateway 2.0 simulation into PR1, gated by config. Promote the test-only inverse codec -(`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, **co-located** in -`src/driver/transport/rntbd/` behind the host feature, reusing the existing token and status -primitives. When a region declares a `thinClientPort`, the emulator binds a Gateway 2.0 listener -that answers `POST /connectivity-probe` with `200`, decodes RNTBD + `thinclient` wire headers into -a logical operation, dispatches through the shared store, and encodes an RNTBD response; the -account topology advertises `thinClient{Readable,Writable}Locations`. +Keep request decoding and response encoding alongside the existing RNTBD client codec in +`azure_data_cosmos_driver`. Expose one feature-gated, high-level emulator entry point that accepts +an RNTBD-framed request, dispatches it through the existing in-memory operation handlers, and +returns an RNTBD-framed response. Token and frame internals remain private to the driver. + +Gateway 2.0 is enabled per region by configuring a thin-client endpoint. Only then does account +discovery advertise `thinClientReadableLocations` and `thinClientWritableLocations`. The +thin-client listener answers `POST /connectivity-probe` and requires HTTP/2. Unsupported RNTBD +semantics are rejected explicitly instead of being silently discarded. ## Consequences -Gateway V1 always works; Gateway 2.0 is opt-in per region, so CI can run the existing suites in -both modes. Because the request pipeline selects Gateway 2.0 purely from the -`thinClient{Readable,Writable}Locations` advertisement, no driver routing change is needed. The -codec halves stay symmetric and co-located for maximal reuse. +The driver remains the single owner of RNTBD wire compatibility. The host depends on a small +unstable adapter rather than public token types. Accounts without thin-client endpoints retain +standard gateway behavior, while configured regions exercise the same discovery and probe flow as +the service. ## Alternatives -- Shipping Gateway 2.0 as a separate later PR was rejected: hosting both from the start lets one - CI pass validate both protocol paths end-to-end. -- Placing the server-side codec in the emulator module was rejected in favor of co-location with - the client codec (Option A), which maximizes reuse and keeps the wire format in one place. +- Implementing a second RNTBD codec in the host crate was rejected because protocol mappings would + drift from the production driver. +- Exposing frame and token types as a general public API was rejected because they are unstable + transport internals. - Always advertising Gateway 2.0 endpoints was rejected: config-gating keeps Gateway V1 as the safe default. diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md new file mode 100644 index 00000000000..4fa555fe0d1 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md @@ -0,0 +1,46 @@ +# ADR-007 — Model outages and failover as dynamic account topology + +**Status:** Proposed +**Date:** 2026-07-14 + +## Context + +The hosted emulator needs to reproduce region outages and write-region changes so SDK routing, +topology refresh, and retry behavior can be tested without a live account. Replication +pause/resume models lag to a target region, but it does not make that region unreachable or remove +it from account discovery. The configured region list and initial write region are otherwise +static. + +## Decision + +Represent availability and write ownership as runtime account-topology state shared by account +discovery and request dispatch: + +- An offline-region set determines which regions appear in readable and writable locations. + Requests sent directly to an offline region fail with `503 Service Unavailable`. +- Single-write accounts maintain a current write-region selection. Changing it updates writable + locations and the write-region guard used by data-plane operations. +- State changes are visible through subsequent account reads; clients observe them through their + normal metadata refresh path rather than an emulator-specific client hook. + +The management REST API mutates this state, while the in-process emulator can use the same store +operations directly. + +## Consequences + +SDK failover behavior is driven by the same account topology contract used with the service. +Replication lag and endpoint outage remain distinct failure models. Existing static behavior is +preserved until the dynamic state is changed. + +## Alternatives + +- Treating replication pause as an outage was rejected because requests can still reach the + region and account discovery still advertises it. +- Rebuilding the emulator for every topology change was rejected because clients could not test + runtime refresh and recovery behavior. +- Adding SDK-specific failover switches was rejected because they would bypass normal account + discovery and routing. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md deleted file mode 100644 index 10875fa5676..00000000000 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_region_offline_failover_primitives.md +++ /dev/null @@ -1,41 +0,0 @@ -# ADR-007 — Add runtime region-offline and write-region failover primitives - -**Status:** Accepted -**Date:** 2026-07-14 - -## Context - -The control plane must support taking a region fully offline and changing the write region at -runtime, to exercise the driver's failover behavior. The store has replication pause/resume and a -per-partition failover flag, but **no** first-class region-offline state and **no** runtime -write-region selection — the write region is fixed to the first configured region in single-write -mode. - -## Decision - -Introduce two new store primitives in PR2, gated behind the host feature: - -- **Region offline/online:** a runtime-mutable set of offline regions. Offline regions are dropped - from the account topology's readable/writable locations and return `503` for data-plane requests. -- **Runtime write-region failover:** a runtime-mutable write-region override (an - `Arc>>`, mirroring the existing per-partition-failover atomic pattern) - consulted by `is_write_region` / `write_region_name` and by the writable-locations topology. - -Both ship with net-new in-process tests and corresponding management REST endpoints. - -## Consequences - -The emulator can reproduce region outage and planned/unplanned write-region failover, so the -driver's re-routing and topology-refresh paths are testable offline. The override defaults to -`None`, preserving today's "first region is the write region" behavior when unused. - -## Alternatives - -- Reusing replication pause/resume to approximate an offline region was rejected: it models lag, - not a region being unreachable, and does not remove the region from topology. -- Making these primitives part of PR1 was rejected: the existing suites do not need them, and they - warrant dedicated tests, so they belong in their own PR. - -## References - -- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md deleted file mode 100644 index 4b6a5997377..00000000000 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_defer_auth_https.md +++ /dev/null @@ -1,56 +0,0 @@ -# ADR-008 — Defer HTTPS and authentication to a dedicated later PR - -**Status:** Accepted -**Date:** 2026-07-14 - -## Context - -Cosmos DB supports key-based auth and Entra ID auth, and production endpoints require HTTPS. -Supporting TLS (certificate provisioning and trust) and authentication introduces real friction -that is orthogonal to standing up the hosted emulator and its control plane. - -## Decision - -PR1 and PR2 host **plaintext HTTP with no authentication**. A dedicated later PR (PR3) adds, -behind an `auth` config block and CLI flags: - -- Optional HTTPS via axum + rustls (`--https --cert --key`); h2c remains the default when off. -- Auth modes: `none` (default), `key` (validate the `Authorization` HMAC against a primary key and - a primary read-only key, matching the service), and `entra` (validate a bearer JWT; see trust - inputs below). -- **HTTPS is required when `key` or `entra` auth is selected.** Enabling either auth mode without - `--https` is a startup error; the process exits with a descriptive message. This prevents account - keys or bearer tokens from being transmitted over plaintext h2c. - -### Entra JWT trust inputs - -An allow-list of object IDs / app IDs is _not_ sufficient to authenticate a JWT. PR3 must define -and validate the following trust inputs before the `entra` mode can be accepted: - -| Input | Flag / config key | Purpose | -|---|---|---| -| JWKS source | `--jwks-uri` or `--jwks-file` | Provides the public keys used to verify the JWT signature. The offline goal recommends `--jwks-file`; `--jwks-uri` is used for online (AAD-connected) deployments. | -| Expected issuer | `--issuer` | The `iss` claim value that the emulator will accept; prevents tokens issued by a different authority from being accepted. | -| Expected audience | `--audience` | The `aud` claim value, typically the emulator's resource URI; prevents tokens issued for other apps from being replayed. | -| Allowed OIDs / app IDs | `--allowed-oid` (repeatable) | After the signature, issuer, and audience are validated, restrict access to the listed object or application IDs. | - -All four inputs must be provided when `entra` mode is enabled; the process exits with a descriptive -error if any are missing. - -## Consequences - -The initial hosting and control-plane work ships without certificate or token friction, and can be -validated over h2c immediately. Auth and HTTPS land as a self-contained, reviewable increment, with -a security-sound design: authenticated modes are gated behind HTTPS, and JWT validation is complete -rather than rely on an allow-list alone. - -## Alternatives - -- Building auth and HTTPS into PR1 was rejected: it enlarges and slows the first deliverable and - couples unrelated concerns. -- Making auth mandatory was rejected: emulator use cases are predominantly local and offline, where - `none` over h2c is the least-friction default. - -## References - -- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md new file mode 100644 index 00000000000..a037433bf1e --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md @@ -0,0 +1,55 @@ +# ADR-008 — Enforce transport security and authentication at the host boundary + +**Status:** Proposed +**Date:** 2026-07-14 + +## Context + +Cosmos DB supports key-based and Microsoft Entra ID authentication, while many local emulator +tests need an authentication-free mode. The in-memory store and operation handlers should not +contain listener-specific TLS, certificate, or token-validation logic. At the same time, +credentials must never be accepted over plaintext transport. + +## Decision + +The host owns transport security and authentication before a request enters the emulator core. +It supports three authentication modes: + +- `none`: no credential validation; allowed over loopback HTTP or HTTPS. +- `key`: validate the Cosmos authorization signature against a primary key or primary read-only + key; HTTPS is required. +- `entra`: validate a bearer token; HTTPS is required. + +Entra validation requires all of the following trust inputs: + +| Input | Purpose | +| --- | --- | +| JWKS URI or local JWKS file | Verify the token signature. | +| Expected issuer | Restrict the accepted token authority. | +| Expected audience | Prevent tokens issued for another resource from being replayed. | +| Allowed object or application IDs | Restrict authenticated principals after cryptographic validation. | + +Selecting an authenticated mode without HTTPS, or omitting required trust inputs, is a startup +error. The emulator core receives only requests that have passed the host policy and remains +independent of certificate and identity libraries. + +## Consequences + +Local tests retain a low-friction no-auth mode. Authenticated scenarios use a security boundary +that is reusable across Gateway V1, Gateway 2.0, and the management API. Offline Entra tests can +use a local JWKS file without weakening signature, issuer, or audience validation. + +## Alternatives + +- Implementing authentication inside operation handlers was rejected because it would couple the + in-process emulator to network-hosting concerns. +- Allowing key or bearer authentication over plaintext was rejected because credentials could be + exposed in transit. +- Checking only an OID/application allow-list was rejected because claims are not trustworthy + until the signature, issuer, and audience have been validated. +- Making authentication mandatory was rejected because many deterministic local tests do not test + identity behavior. + +## References + +- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md deleted file mode 100644 index f0052fa9d50..00000000000 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_ci_reuse_existing_suites.md +++ /dev/null @@ -1,39 +0,0 @@ -# ADR-009 — Validate via the existing suites over a new `emulator_inmemory` cfg - -**Status:** Accepted -**Date:** 2026-07-14 - -## Context - -The hosted emulator is only meaningful if it faithfully serves the real wire protocol. The most -convincing validation is the existing `emulator` / `emulator_vnext` integration suites passing -against it. The suites are gated at runtime by a `test_category` cfg set via `RUSTFLAGS`. - -## Decision - -Add a new `test_category = "emulator_inmemory"` cfg (registered in the `build.rs` of both -`azure_data_cosmos` and `azure_data_cosmos_driver`) and add it to the existing emulator suites' -`#[cfg_attr(...)]` gates and ignore messages, preserving intentional legacy-only exclusions. Extend -`sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary with a -provisioning config, wait for `GET /health`, and point `AZURE_COSMOS_CONNECTION_STRING` at the hosted endpoint. Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` (modeled on -`Cosmos_vnext_emulator`) that runs the existing suites in **both** Gateway V1 and Gateway 2.0 -(RNTBD) modes. This CI pass replaces a standalone HTTP/2 validation spike. - -## Consequences - -The same battle-tested suites validate h2c and RNTBD end-to-end, in both gateway modes, without -new bespoke tests. Starting non-blocking (`ContinueOnError`) surfaces failures as "succeeded with -issues" while the host stabilizes. - -## Alternatives - -- A separate standalone h2c validation spike was rejected: running the real suites over the hosted - emulator is stronger and is exactly what the combined PR delivers. -- A bespoke `emulator_inmemory`-only test set was rejected: reusing the existing suites maximizes - coverage and avoids drift. -- Reusing the `emulator_vnext` cfg was rejected: its behavioral-divergence skips do not match the - in-memory emulator, so a distinct cfg is cleaner. - -## References - -- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md deleted file mode 100644 index 152907aa049..00000000000 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/010_pr_sequencing.md +++ /dev/null @@ -1,39 +0,0 @@ -# ADR-010 — PR sequencing - -**Status:** Accepted -**Date:** 2026-07-14 - -## Context - -The work spans a new crate, host feature surface, data-plane hosting for two gateway flavors, a -control-plane API, new store primitives, CI, and eventually auth/HTTPS. It needs to be split into -reviewable increments that each land something coherent. - -## Decision - -Sequence the work as three PRs: - -- **PR1** — the new host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 - (config-gated); the management REST API over the control-plane actions that map to existing store - methods; and CI running the existing suites against the hosted emulator in both gateway modes. -- **PR2** — the new store primitives (region offline/online, runtime write-region failover) with - net-new tests and their management endpoints. -- **PR3** — optional HTTPS and authentication. - -## Consequences - -PR1 is independently meaningful because the CI leg validates it end-to-end; it does not depend on -the new primitives, which the existing suites do not exercise. PR2 and PR3 are self-contained, -testable increments. Gateway 2.0 is folded into PR1 so a single CI pass validates both protocol -paths. - -## Alternatives - -- Splitting hosting and CI into separate PRs was rejected: hosting without the CI validation proves - nothing, so they ship together. -- Deferring Gateway 2.0 to its own PR was rejected: hosting both gateways from the start lets one CI - pass cover both (ADR-006). - -## References - -- Plan & summary: ../plan.md diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 1d2e95ecef6..357c37c69a3 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -518,24 +518,31 @@ block and CLI flags: check its object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). Splitting auth/HTTPS into its own PR keeps certificate and token friction out of the initial -hosting and control-plane work (ADR-008). +hosting and control-plane work ([ADR-008](adr/008_transport_security_and_authentication.md)). --- ## 11. Architecture Decision Records -| ADR | Decision | -| -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| [ADR-001](adr/001_separate_host_crate.md) | Host in a separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. | -| [ADR-002](adr/002_port_per_region.md) | Model each region as a distinct localhost port; one shared store. | -| [ADR-003](adr/003_cleartext_http2.md) | Serve cleartext HTTP/2 (h2c) and reuse the driver's existing prior-knowledge probe. | -| [ADR-004](adr/004_management_rest_api.md) | Expose emulator-only control-plane actions via a separate management REST API. | -| [ADR-005](adr/005_json_config_startup_seed.md) | Drive startup topology and seed data from a JSON config file; defer YAML. | -| [ADR-006](adr/006_gateway_v2_rntbd.md) | Simulate Gateway 2.0 by promoting the test-only inverse RNTBD codec to production, config-gated. | -| [ADR-007](adr/007_region_offline_failover_primitives.md) | Add runtime region-offline and write-region failover store primitives (PR2). | -| [ADR-008](adr/008_defer_auth_https.md) | Defer HTTPS and authentication to a dedicated later PR. | -| [ADR-009](adr/009_ci_reuse_existing_suites.md) | Validate via the existing suites over a new `emulator_inmemory` cfg, in both gateway modes. | -| [ADR-010](adr/010_pr_sequencing.md) | Sequence the work as PR1 (hosting + control plane + CI), PR2 (primitives), PR3 (auth/HTTPS). | +- [ADR-001](adr/001_separate_host_crate.md): Host in a separate `publish = false` binary crate; + keep the emulator in the driver behind a host feature. +- [ADR-002](adr/002_port_per_region.md): Model each region as a distinct localhost port backed by + one shared store. +- [ADR-003](adr/003_cleartext_http2.md): Support h2c on emulator endpoints while retaining + HTTPS-only production routing. +- [ADR-004](adr/004_management_rest_api.md): Expose emulator-only control-plane actions through a + separate management REST API. +- [ADR-005](adr/005_json_config_startup_seed.md): Keep canonical startup configuration in + host-owned JSON and seed through the data plane. +- [ADR-006](adr/006_gateway_v2_rntbd.md): Keep RNTBD framing private to the driver behind a + high-level hosted-emulator adapter. +- [ADR-007](adr/007_dynamic_account_topology.md): Model outages and write-region failover as + dynamic account-topology state. +- [ADR-008](adr/008_transport_security_and_authentication.md): Enforce transport security and + authentication at the host boundary. + +Delivery sequencing, CI rollout, and temporary test exclusions remain in this plan. They are not +architecture decisions and are intentionally excluded from the ADR directory. --- From 5be250eb7372466f8f120b7b997ad8b706711d10 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 14 Jul 2026 16:30:39 +0000 Subject: [PATCH 23/44] Unlinking links --- .../docs/adr/001_separate_host_crate.md | 2 +- .../docs/adr/002_port_per_region.md | 2 +- .../docs/adr/003_cleartext_http2.md | 5 +- .../docs/adr/004_management_rest_api.md | 2 +- .../docs/adr/005_json_config_startup_seed.md | 2 +- .../docs/adr/006_gateway_v2_rntbd.md | 4 +- .../docs/adr/007_dynamic_account_topology.md | 2 +- ...8_transport_security_and_authentication.md | 2 +- .../azure_data_cosmos_emulator/docs/plan.md | 74 ++++++++++--------- 9 files changed, 50 insertions(+), 45 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md index 98211d41c61..c36a2d33591 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md @@ -32,4 +32,4 @@ prefix), so stable builds and the public API are unaffected. ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md index 43544d9ac3c..62d9ab14d09 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md @@ -31,4 +31,4 @@ serve every port. Region gateway URLs in the config are plain `http://127.0.0.1: ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md index 97804071303..21d6bc43dfe 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md @@ -37,5 +37,6 @@ carry credentials require TLS, as described by ADR-008. ## References -- Plan & summary: ../plan.md -- Transport pipeline spec: ../../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` +- Transport pipeline spec: + `sdk/cosmos/azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md index 3a38c54278d..82798414f9f 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md @@ -32,4 +32,4 @@ and the management API never collides with Cosmos paths because it lives on a se ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md index c082bcb3266..a13a48df0ce 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md @@ -40,4 +40,4 @@ particular parser implementation. ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md index 4a09fec5671..ad8cd194e02 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md @@ -43,5 +43,5 @@ the service. ## References -- Plan & summary: ../plan.md -- Gateway 2.0 spec: ../../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` +- Gateway 2.0 spec: `sdk/cosmos/azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md index 4fa555fe0d1..b78a72effc4 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md @@ -43,4 +43,4 @@ preserved until the dynamic state is changed. ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md index a037433bf1e..87da2d0bdb2 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md @@ -52,4 +52,4 @@ use a local JWKS file without weakening signature, issuer, or audience validatio ## References -- Plan & summary: ../plan.md +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 357c37c69a3..1ea6598adf2 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -9,19 +9,19 @@ ## Table of Contents -1. [Goals & Motivation](#1-goals--motivation) -2. [Scope, Phasing & Feature Gating](#2-scope-phasing--feature-gating) -3. [Architecture Overview](#3-architecture-overview) -4. [Configuration File](#4-configuration-file) -5. [Data-Plane Hosting (Gateway V1 & Gateway 2.0)](#5-data-plane-hosting-gateway-v1--gateway-20) -6. [Management REST API (Control Plane)](#6-management-rest-api-control-plane) -7. [Rust Public API Surface](#7-rust-public-api-surface) -8. [HTTP/2 & Transport Notes](#8-http2--transport-notes) -9. [CI Integration](#9-ci-integration) -10. [Authentication & HTTPS (Deferred)](#10-authentication--https-deferred) -11. [Architecture Decision Records](#11-architecture-decision-records) -12. [Open Questions & Future Work](#12-open-questions--future-work) -13. [References](#13-references) +1. Goals & Motivation +2. Scope, Phasing & Feature Gating +3. Architecture Overview +4. Configuration File +5. Data-Plane Hosting (Gateway V1 & Gateway 2.0) +6. Management REST API (Control Plane) +7. Rust Public API Surface +8. HTTP/2 & Transport Notes +9. CI Integration +10. Authentication & HTTPS (Deferred) +11. Architecture Decision Records +12. Open Questions & Future Work +13. References --- @@ -518,28 +518,30 @@ block and CLI flags: check its object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). Splitting auth/HTTPS into its own PR keeps certificate and token friction out of the initial -hosting and control-plane work ([ADR-008](adr/008_transport_security_and_authentication.md)). +hosting and control-plane work. See ADR-008 in +`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md`. --- ## 11. Architecture Decision Records -- [ADR-001](adr/001_separate_host_crate.md): Host in a separate `publish = false` binary crate; - keep the emulator in the driver behind a host feature. -- [ADR-002](adr/002_port_per_region.md): Model each region as a distinct localhost port backed by - one shared store. -- [ADR-003](adr/003_cleartext_http2.md): Support h2c on emulator endpoints while retaining - HTTPS-only production routing. -- [ADR-004](adr/004_management_rest_api.md): Expose emulator-only control-plane actions through a - separate management REST API. -- [ADR-005](adr/005_json_config_startup_seed.md): Keep canonical startup configuration in - host-owned JSON and seed through the data plane. -- [ADR-006](adr/006_gateway_v2_rntbd.md): Keep RNTBD framing private to the driver behind a - high-level hosted-emulator adapter. -- [ADR-007](adr/007_dynamic_account_topology.md): Model outages and write-region failover as - dynamic account-topology state. -- [ADR-008](adr/008_transport_security_and_authentication.md): Enforce transport security and - authentication at the host boundary. +- ADR-001 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md`): Host in a + separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. +- ADR-002 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md`): Model each + region as a distinct localhost port backed by one shared store. +- ADR-003 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md`): Support h2c on + emulator endpoints while retaining HTTPS-only production routing. +- ADR-004 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md`): Expose + emulator-only control-plane actions through a separate management REST API. +- ADR-005 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md`): Keep + canonical startup configuration in host-owned JSON and seed through the data plane. +- ADR-006 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md`): Keep RNTBD + framing private to the driver behind a high-level hosted-emulator adapter. +- ADR-007 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md`): Model + outages and write-region failover as dynamic account-topology state. +- ADR-008 + (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md`): + Enforce transport security and authentication at the host boundary. Delivery sequencing, CI rollout, and temporary test exclusions remain in this plan. They are not architecture decisions and are intentionally excluded from the ADR directory. @@ -561,8 +563,10 @@ architecture decisions and are intentionally excluded from the ADR directory. ## 13. References -- In-memory emulator spec: `../../azure_data_cosmos/docs/in-memory-emulator-spec.md` -- Gateway 2.0 spec: `../../azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` -- Transport pipeline spec: `../../azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` -- RNTBD codec: `../../azure_data_cosmos_driver/src/driver/transport/rntbd/` -- Cosmos test setup: `../../eng/scripts/Invoke-CosmosTestSetup.ps1` +- In-memory emulator spec: + `sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md` +- Gateway 2.0 spec: `sdk/cosmos/azure_data_cosmos_driver/docs/GATEWAY_V2_SPEC.md` +- Transport pipeline spec: + `sdk/cosmos/azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md` +- RNTBD codec: `sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/` +- Cosmos test setup: `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` From f3577b8567c6846e547ce15ec9498d7686dcd3ca Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:32:54 +0000 Subject: [PATCH 24/44] Updating ADRs --- ...1_build_memory_backed_sdk_test_emulator.md | 74 +++++++++++++++++++ ...st_crate.md => 002_separate_host_crate.md} | 2 +- ...t_per_region.md => 003_port_per_region.md} | 2 +- ...artext_http2.md => 004_cleartext_http2.md} | 4 +- ...rest_api.md => 005_management_rest_api.md} | 2 +- ...eed.md => 006_json_config_startup_seed.md} | 2 +- ...ay_v2_rntbd.md => 007_gateway_v2_rntbd.md} | 2 +- ...ogy.md => 008_dynamic_account_topology.md} | 2 +- ..._transport_security_and_authentication.md} | 2 +- .../azure_data_cosmos_emulator/docs/plan.md | 35 +++++---- 10 files changed, 102 insertions(+), 25 deletions(-) create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{001_separate_host_crate.md => 002_separate_host_crate.md} (95%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{002_port_per_region.md => 003_port_per_region.md} (95%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{003_cleartext_http2.md => 004_cleartext_http2.md} (93%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{004_management_rest_api.md => 005_management_rest_api.md} (96%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{005_json_config_startup_seed.md => 006_json_config_startup_seed.md} (96%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{006_gateway_v2_rntbd.md => 007_gateway_v2_rntbd.md} (97%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{007_dynamic_account_topology.md => 008_dynamic_account_topology.md} (96%) rename sdk/cosmos/azure_data_cosmos_emulator/docs/adr/{008_transport_security_and_authentication.md => 009_transport_security_and_authentication.md} (97%) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md new file mode 100644 index 00000000000..b067a2f4007 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md @@ -0,0 +1,74 @@ +# ADR-001 — Build a memory-backed emulator host for Cosmos DB SDK testing + +**Status:** Accepted +**Date:** 2026-07-15 + +## Context + +Cosmos DB SDKs need deterministic tests for behavior driven by account and partition topology: +regional routing, endpoint outages and recovery, write-region changes, replication lag, physical +partition split and merge, and Gateway V1 versus Gateway 2.0 transport selection. These conditions +are difficult to create repeatedly in a live account and often cannot be triggered at the exact +point required by a test. + +The existing Cosmos DB emulators are useful for general service compatibility, but they do not +provide the fine-grained, runtime topology controls required by SDK routing and resiliency tests. +They also introduce platform, container, startup-time, and certificate dependencies that make +large deterministic test matrices more expensive and less predictable. + +A network-accessible host is required so Rust, Java, .NET, Python, and other SDKs can exercise their +real HTTP stacks and wire protocols against the same controllable topology model. + +## Decision + +Build an open-source, memory-backed Cosmos DB emulator host whose primary purpose is testing SDK +routing, topology refresh, retry, failover, and transport behavior. The host exposes supported +Cosmos DB data-plane and metadata contracts over network ports and exposes an emulator-specific +management API for deterministic topology changes. + +Optimize the emulator for topology control, deterministic behavior, fast startup, and useful wire +fidelity. It is not a complete Cosmos DB service implementation and is not intended to become one. +In particular: + +- It is an SDK engineering and test tool, not a supported customer product. Because it is public + and open source, customers may use it, but it provides no service compatibility, durability, + performance, availability, or support guarantees. +- It stores all state in memory and does not provide persistence or production data safety. +- It implements only the data-plane operations needed to provision fixtures and observe SDK + behavior under simulated topology conditions. +- Query support is deliberately limited to the subset needed by SDK scenarios. Full SQL query + semantics, broad query compatibility, and other complex data-plane features are non-goals. +- Unsupported operations or protocol semantics fail explicitly rather than silently approximating + behavior that could invalidate an SDK test. + +## Consequences + +Topology and transport fidelity take priority over breadth of service emulation. New capabilities +are added when they enable a concrete SDK test scenario, not to pursue general Cosmos DB feature +parity. + +Tests can run quickly and deterministically without Azure resources or heavyweight emulator +infrastructure. Cross-language SDKs can share the same network-visible behavior while retaining +their real client pipelines. + +The project must document supported behavior and known divergences clearly so its intentionally +limited scope is not mistaken for customer-grade service emulation. + +## Alternatives + +- Live Cosmos DB accounts were rejected as the primary test mechanism because they add cost, + provisioning time, environmental variability, and cannot deterministically expose every + topology transition or failure at a chosen point in a test. +- Existing Cosmos DB emulators were rejected as the sole mechanism because they do not expose the + runtime topology controls required for region, replication, split, merge, and failover scenarios, + and they carry additional platform and deployment dependencies. +- SDK-local mocks were rejected as the shared solution because they bypass real network and wire + behavior, duplicate semantics in every language, and cannot validate Gateway 2.0 framing or + cross-SDK compatibility. +- Expanding the project into a full customer-facing Cosmos DB emulator was rejected because it + would shift effort away from deterministic SDK topology testing and create an impractical service + compatibility and support commitment. + +## References + +- Plan & summary: `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md` diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md similarity index 95% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md index c36a2d33591..ec3ac56bd5f 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md @@ -1,4 +1,4 @@ -# ADR-001 — Host in a separate binary crate; keep the emulator in the driver +# ADR-002 — Host in a separate binary crate; keep the emulator in the driver **Status:** Accepted **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md similarity index 95% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md index 62d9ab14d09..876f47de3e6 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md @@ -1,4 +1,4 @@ -# ADR-002 — Model each region as a distinct localhost port +# ADR-003 — Model each region as a distinct localhost port **Status:** Accepted **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_cleartext_http2.md similarity index 93% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_cleartext_http2.md index 21d6bc43dfe..e7bfc235f0c 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_cleartext_http2.md @@ -1,4 +1,4 @@ -# ADR-003 — Support cleartext HTTP/2 for local emulator endpoints +# ADR-004 — Support cleartext HTTP/2 for local emulator endpoints **Status:** Accepted **Date:** 2026-07-14 @@ -25,7 +25,7 @@ traffic uses HTTP/2 or falls back to HTTP/1.1. Gateway 2.0 can be exercised locally without certificates. Plaintext traffic is limited to explicit emulator hosts, preserving the production transport boundary. Authentication modes that -carry credentials require TLS, as described by ADR-008. +carry credentials require TLS, as described by ADR-009. ## Alternatives diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md similarity index 96% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md index 82798414f9f..a14b65ce0a3 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md @@ -1,4 +1,4 @@ -# ADR-004 — Expose control-plane actions via a separate management REST API +# ADR-005 — Expose control-plane actions via a separate management REST API **Status:** Accepted **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md similarity index 96% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md index a13a48df0ce..420793fb406 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md @@ -1,4 +1,4 @@ -# ADR-005 — Drive startup topology and seed data from a JSON config file +# ADR-006 — Drive startup topology and seed data from a JSON config file **Status:** Accepted **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md similarity index 97% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md index ad8cd194e02..6e569070189 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md @@ -1,4 +1,4 @@ -# ADR-006 — Keep hosted Gateway 2.0 framing in the driver +# ADR-007 — Keep hosted Gateway 2.0 framing in the driver **Status:** Accepted **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_dynamic_account_topology.md similarity index 96% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_dynamic_account_topology.md index b78a72effc4..67ae32cc011 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_dynamic_account_topology.md @@ -1,4 +1,4 @@ -# ADR-007 — Model outages and failover as dynamic account topology +# ADR-008 — Model outages and failover as dynamic account topology **Status:** Proposed **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md similarity index 97% rename from sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md rename to sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md index 87da2d0bdb2..e71f168b552 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md @@ -1,4 +1,4 @@ -# ADR-008 — Enforce transport security and authentication at the host boundary +# ADR-009 — Enforce transport security and authentication at the host boundary **Status:** Proposed **Date:** 2026-07-14 diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 1ea6598adf2..fa59f9d0562 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -143,7 +143,7 @@ sdk/cosmos/azure_data_cosmos_emulator/ A single JSON file, referenced with `--config`, describes the account topology, the databases and containers to create, and optional seed items — all applied on startup. The management REST -API can further modify state at runtime. (YAML support is deferred; see ADR-005.) +API can further modify state at runtime. (YAML support is deferred; see ADR-006.) ### 4.1 Sample @@ -233,7 +233,7 @@ Enabled per region by setting `thinClientPort`. When enabled: The driver already owns the client-side RNTBD codec (`RntbdRequestFrame::write`, `RntbdResponse::read`). This work promotes the currently test-only inverse halves (`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, co-located in -`src/driver/transport/rntbd/` behind the host feature (ADR-006). Because the driver's request +`src/driver/transport/rntbd/` behind the host feature (ADR-007). Because the driver's request pipeline decides Gateway 2.0 purely from whether the endpoint advertises a Gateway 2.0 URL, advertising it plus serving RNTBD is sufficient — no driver routing change is required. @@ -487,7 +487,7 @@ HTTP/2 is a hard requirement for Gateway 2.0. The relevant driver behavior **alr prior-knowledge connections when axum is compiled with its non-default `http2` feature. The host must enable that feature explicitly. With it enabled, the existing probe negotiates HTTP/2 against the host and no mandatory driver change is required. A targeted change to the incompatibility -matcher is a **contingency** only if end-to-end validation reveals a gap (ADR-003). +matcher is a **contingency** only if end-to-end validation reveals a gap (ADR-004). --- @@ -518,29 +518,32 @@ block and CLI flags: check its object ID / app ID against an allow-list supplied in config or via `--allowed-oid`). Splitting auth/HTTPS into its own PR keeps certificate and token friction out of the initial -hosting and control-plane work. See ADR-008 in -`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md`. +hosting and control-plane work. See ADR-009 in +`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md`. --- ## 11. Architecture Decision Records -- ADR-001 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_separate_host_crate.md`): Host in a +- ADR-001 + (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md`): + Build a memory-backed host for deterministic Cosmos DB SDK topology and transport testing. +- ADR-002 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md`): Host in a separate `publish = false` binary crate; keep the emulator in the driver behind a host feature. -- ADR-002 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_port_per_region.md`): Model each +- ADR-003 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md`): Model each region as a distinct localhost port backed by one shared store. -- ADR-003 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_cleartext_http2.md`): Support h2c on +- ADR-004 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_cleartext_http2.md`): Support h2c on emulator endpoints while retaining HTTPS-only production routing. -- ADR-004 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/004_management_rest_api.md`): Expose +- ADR-005 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md`): Expose emulator-only control-plane actions through a separate management REST API. -- ADR-005 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_json_config_startup_seed.md`): Keep +- ADR-006 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md`): Keep canonical startup configuration in host-owned JSON and seed through the data plane. -- ADR-006 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_gateway_v2_rntbd.md`): Keep RNTBD +- ADR-007 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md`): Keep RNTBD framing private to the driver behind a high-level hosted-emulator adapter. -- ADR-007 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_dynamic_account_topology.md`): Model +- ADR-008 (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_dynamic_account_topology.md`): Model outages and write-region failover as dynamic account-topology state. -- ADR-008 - (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/008_transport_security_and_authentication.md`): +- ADR-009 + (`sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md`): Enforce transport security and authentication at the host boundary. Delivery sequencing, CI rollout, and temporary test exclusions remain in this plan. They are not @@ -551,10 +554,10 @@ architecture decisions and are intentionally excluded from the ADR directory. ## 12. Open Questions & Future Work - **YAML config.** Deferred; `serde_yaml` is unmaintained, so a maintained crate would be added - in a follow-up if demand warrants (ADR-005). + in a follow-up if demand warrants (ADR-006). - **h2c fallback hardening.** If validation shows the driver does not cleanly fall back from h2c to HTTP/1.1 against a cleartext HTTP/1.1-only server, broaden `has_explicit_http2_incompatibility` - (ADR-003). Not required while the host always serves h2c. + (ADR-004). Not required while the host always serves h2c. - **Additional control-plane ops.** Throttling toggles, forced session-not-available, and replication-delay overrides could be surfaced through the management API later if cross-SDK tests need them. From 37f00497efaebbd7893b7cd303b3633dcb7b92d1 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:43:12 +0000 Subject: [PATCH 25/44] Addressing code review comments --- .../docs/adr/002_separate_host_crate.md | 15 +- .../docs/adr/003_port_per_region.md | 52 +++++-- .../docs/adr/005_management_rest_api.md | 28 ++++ .../docs/adr/006_json_config_startup_seed.md | 5 + .../azure_data_cosmos_emulator/docs/plan.md | 137 ++++++++++++------ 5 files changed, 180 insertions(+), 57 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md index ec3ac56bd5f..045ab19a3b3 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md @@ -14,21 +14,22 @@ exposing large swaths of internal surface. Add a new `publish = false` binary crate `azure_data_cosmos_emulator` that hosts the emulator. The emulator implementation stays in the driver; the driver exposes a small, additional **public** -surface behind a feature flag (`__internal_in_memory_emulator_host`) that the host crate enables -automatically through its dependency declaration. +surface behind the existing `__internal_in_memory_emulator` feature. The host crate enables that +feature automatically through its dependency declaration. ## Consequences The host crate stays thin (CLI, HTTP listeners, config, management API). The emulator keeps full -access to driver internals. The extra surface is opt-in and clearly non-SemVer (the `__internal_` -prefix), so stable builds and the public API are unaffected. +access to driver internals. The externally reachable emulator surface remains opt-in and clearly +non-SemVer (the `__internal_` prefix), so stable builds are unaffected. ## Alternatives -- Extracting the emulator into its own library crate was rejected: it would force a large, unstable slice of driver internals to become public. -- Growing the existing test-only feature to cover hosting was rejected: it would entangle - in-process test wiring with server-only concerns. + internal feature already communicates the same stability boundary. A second feature would add + feature-matrix complexity without providing a meaningful compatibility guarantee. +- Adding a second host-specific feature was rejected because the existing emulator feature already + owns this internal surface and the split would add no useful granularity. ## References diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md index 876f47de3e6..41138dfe632 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md @@ -1,4 +1,4 @@ -# ADR-003 — Model each region as a distinct localhost port +# ADR-003 — Expose each region through a distinct discoverable endpoint **Status:** Accepted **Date:** 2026-07-14 @@ -7,27 +7,61 @@ The driver reads account topology and routes subsequent requests to each region's `databaseAccountEndpoint`, which must be an independently reachable network endpoint. The store -already resolves a request's region by `(scheme, host, port)`. +already resolves a request's region by `(scheme, host, port)`. Fixed ports are convenient for +interactive use but collide when tests run concurrently on shared CI agents. ## Decision -Bind one gateway listener per region on a distinct `127.0.0.1:{port}`. A single shared -`EmulatorStore` backs every listener; the region is resolved per request from the `Host` header. -A single-region account is simply one port. Gateway 2.0 adds an optional second RNTBD port per -region. +Bind one gateway listener per region to a distinct loopback endpoint. A single shared +`EmulatorStore` backs every listener; the region is resolved per request from the full URL. A +single-region account has one standard gateway endpoint. Gateway 2.0 adds an optional second RNTBD +endpoint per region. + +Every configured port is optional and defaults to `0`, which asks the operating system to assign +an available port. An explicit non-zero port remains available for interactive scenarios that +need a stable endpoint. Account discovery advertises the **actual bound URLs**, never the requested +port value. + +After all listeners are bound, the host writes one machine-readable JSON `ready` record to stdout. +Diagnostic logs go to stderr so automation can parse stdout without log filtering. The record +contains the management endpoint, the hub account endpoint, and every region's standard gateway +and optional Gateway 2.0 endpoint. For example: + +```json +{ + "event": "ready", + "managementEndpoint": "http://127.0.0.1:49150/", + "accountEndpoint": "http://127.0.0.1:49151/", + "regions": [ + { + "name": "East US", + "gatewayEndpoint": "http://127.0.0.1:49151/", + "gateway20Endpoint": "http://127.0.0.1:49152/" + } + ] +} +``` + +The management account endpoint returns the same resolved topology after startup. Consumers use +the complete URLs from either contract and do not reconstruct endpoints from configured ports. ## Consequences Multi-region topologies map cleanly onto the driver's existing endpoint routing with no new region-resolution mechanism. The store models all regions internally, so one process and one store -serve every port. Region gateway URLs in the config are plain `http://127.0.0.1:{port}` values. +serve every endpoint. Concurrent test hosts avoid port collisions by default. Supplying full URLs +to clients preserves the option to use host-based routing or non-loopback bindings in the future +without changing the discovery contract. ## Alternatives -- A single port differentiated by `Host` header (e.g. `eastus.localhost`) was rejected: it relies - on client-side DNS/hosts trickery and does not match how the driver dials distinct endpoints. +- A single port differentiated by `Host` header (e.g. `eastus.localhost`) was not selected because + it relies on client-side DNS/hosts setup. Returning full URLs leaves that implementation option + open for the future. - One process per region was rejected: it would fragment the shared store and complicate cross-region replication and failover simulation. +- Requiring fixed ports was rejected because parallel tests cannot reliably coordinate port + ownership on shared agents. ## References diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md index a14b65ce0a3..87df922c850 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md @@ -17,11 +17,36 @@ Serve the emulator-only control-plane actions through a dedicated **management R own port, distinct from the Cosmos wire protocol. Gateway-native lifecycle operations are **not** duplicated there; callers use the standard Cosmos endpoints (or startup config seeding) for those. +Topology-changing actions such as partition split and merge are represented as long-running +operation resources. Creating one returns `202 Accepted` and an operation ID. The operation exposes +these phases: + +1. `Preparing`: the existing partition topology remains available. +2. `Swapping`: source partition requests return `410/1007`; replacement partitions are not yet + visible. +3. `Succeeded`: source partitions are gone and replacement partitions are visible. +4. `Failed`: a terminal error is available on the operation resource. + +Each operation selects one lock/progression mode: + +- `instant` (default): progress automatically through all phases without a guaranteed observable + `Swapping` window. +- `delayed`: enter `Swapping` automatically, remain there for `lockDurationMs`, then complete. +- `manual`: remain in each non-terminal phase until `POST /operations/{operationId}/advance` moves + the operation forward by one phase. + +`lockDurationMs` is required and positive for `delayed`; it is rejected for `instant` and `manual`. +Advancing an automatic or terminal operation returns `409 Conflict`. The operation state machine, +not a generic lock API, owns partition locking so tests cannot create a lock state unrelated to an +actual topology transition. + ## Consequences Other SDKs and operators drive emulator-specific behavior (split, merge, failover, offline) over HTTP without an in-process handle. The Cosmos data plane stays a pure wire-protocol surface, and the management API never collides with Cosmos paths because it lives on a separate port. +Manual advancement gives tests deterministic assertions before, during, and after a topology +change without timing races. ## Alternatives @@ -29,6 +54,9 @@ and the management API never collides with Cosmos paths because it lives on a se rejected as more collision-prone and less discoverable than a dedicated port. - Duplicating database/container CRUD in the management API was rejected: those are already expressible through the gateway contract. +- Exposing generic partition lock/unlock endpoints was rejected because they could create states + that do not correspond to a real topology operation. +- Relying only on lock duration was rejected because timing-based tests are inherently racy. ## References diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md index 420793fb406..5335b3308b6 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/006_json_config_startup_seed.md @@ -18,6 +18,11 @@ Seed documents are created through the normal write path — one synthesized cre item through `execute_request` — so EPK routing, RU accounting, and replication match client-issued writes. The management REST API can further modify state at runtime. +Listener ports are optional configuration hints. Missing values and `0` request OS-assigned ports; +the runtime endpoint contract is the JSON `ready` record and management account response described +by ADR-003. Public configuration uses `gateway20` terminology. Literal `thinClient*` names are +reserved for Cosmos account-topology fields and wire headers. + JSON is the canonical configuration representation. Additional syntaxes may be introduced only as parsers that map to the same host-owned configuration model; they must not create a second set of configuration semantics. diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index fa59f9d0562..31f76385ca5 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -3,7 +3,7 @@ **Status:** Draft / Plan (not yet implemented) **Date:** 2026-07-14 **Crates:** `azure_data_cosmos_emulator` (new binary host), `azure_data_cosmos_driver` (emulator core, behind a feature) -**Feature gate:** `__internal_in_memory_emulator_host` (non-SemVer, host-only surface) +**Feature gate:** `__internal_in_memory_emulator` (non-SemVer emulator surface) --- @@ -60,19 +60,18 @@ The work ships across three pull requests. | PR | Contents | Feature/cfg | | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | -| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator_host`, `test_category = "emulator_inmemory"` | +| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator`, `test_category = "emulator_inmemory"` | | **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | | **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | ### Feature gating -- The emulator core remains behind the existing `__internal_in_memory_emulator` feature. -- A new feature `__internal_in_memory_emulator_host = ["__internal_in_memory_emulator"]` gates - the additional **host-only** public surface (the server-side RNTBD codec halves and the new - control-plane primitives). The host crate enables it automatically via its dependency - declaration, so consumers never set it by hand. -- Both features use the `__internal_` prefix: this surface is **not** part of any SemVer - contract and may change at any time. +- The emulator core and the public surface required by the host remain behind the existing + `__internal_in_memory_emulator` feature. +- The host crate enables the feature automatically via its dependency declaration, so consumers + of the host never set it by hand. +- The `__internal_` prefix means the surface is **not** part of the driver's SemVer contract and + may change at any time. --- @@ -85,11 +84,11 @@ flowchart LR subgraph HOST["azure_data_cosmos_emulator (binary)"] direction TB - EG["East US gateway :8081"] - ET["East US Gateway 2.0 :8444"] - WG["West US gateway :8082"] - WT["West US Gateway 2.0 :8445"] - MGMT["Management API :9090"] + EG["East US gateway
(assigned endpoint)"] + ET["East US Gateway 2.0
(assigned endpoint)"] + WG["West US gateway
(assigned endpoint)"] + WT["West US Gateway 2.0
(assigned endpoint)"] + MGMT["Management API
(assigned endpoint)"] STORE["Single shared store
Arc<InMemoryEmulatorHttpClient> owns Arc<EmulatorStore>"] EG --> STORE ET --> STORE @@ -114,7 +113,7 @@ flowchart LR - **Data-plane bridge.** Each gateway listener rebuilds an `azure_core::http::Request` from the incoming HTTP request and calls `InMemoryEmulatorHttpClient::execute_request`, then converts the `AsyncRawResponse` back to an HTTP response. -- **Gateway 2.0 (optional).** When a region declares a `thinClientPort`, the host binds a +- **Gateway 2.0 (optional).** When a region enables `gateway20`, the host binds a Gateway 2.0 listener that answers the connectivity probe and speaks RNTBD; the store's account topology then advertises `thinClient{Readable,Writable}Locations`. - **Management port.** A dedicated port serves the control-plane REST API, kept separate from the @@ -156,15 +155,15 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-006. "perPartitionFailover": false, "throttling": false, "regions": [ - { "name": "East US", "gatewayPort": 8081, "thinClientPort": 8444, "regionId": 0 }, - { "name": "West US", "gatewayPort": 8082, "thinClientPort": 8445, "regionId": 1 } + { "name": "East US", "gatewayPort": 0, "gateway20Port": 0, "regionId": 0 }, + { "name": "West US", "gatewayPort": 0, "gateway20Port": 0, "regionId": 1 } ], "replication": { "minDelayMs": 20, "maxDelayMs": 50, "maxBufferedReplications": 10000 }, "replicationOverrides": [ { "source": "East US", "target": "West US", "minDelayMs": 200, "maxDelayMs": 500 } ] }, - "management": { "port": 9090 }, + "management": { "port": 0 }, "databases": [ { "id": "testdb", @@ -193,12 +192,12 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-006. | `account.consistency` | enum | `Strong \| BoundedStaleness \| Session \| ConsistentPrefix \| Eventual`. | | `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | | `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | -| `account.regions[].gatewayPort` | u16 | Gateway V1 (JSON REST) port. Region endpoint = `http://127.0.0.1:{gatewayPort}`. | -| `account.regions[].thinClientPort` | u16, optional | When present, enables Gateway 2.0 simulation for the region. | +| `account.regions[].gatewayPort` | u16, optional | Standard gateway port. Missing or `0` requests an OS-assigned port. | +| `account.regions[].gateway20Port` | u16, optional | Enables Gateway 2.0. Missing disables it; `0` requests an OS-assigned port. | | `account.regions[].regionId` | u64, optional | Auto-assigned by position when omitted. | | `account.replication` | object | Default replication delay + buffer cap. | | `account.replicationOverrides[]` | array | Per source→target replication overrides. | -| `management.port` | u16 | Control-plane REST API port. | +| `management.port` | u16, optional | Management API port. Missing or `0` requests an OS-assigned port. | | `databases[].containers[].partitionKey` | object | Standard Cosmos partition key definition (`paths`, `kind`, `version`). | | `databases[].containers[].partitionCount` | u32 | Initial physical partition count. | | `databases[].containers[].throughput` | u32 | Provisioned RU/s (drives throttling when enabled). | @@ -208,6 +207,26 @@ Seed items are created through the same request path as real writes (a synthesiz request per item), so EPK routing, RU accounting, and replication behave identically to client-issued writes. +After binding all listeners, the host writes one JSON `ready` record to stdout. It contains the +resolved management endpoint, hub account endpoint, and all regional standard-gateway and +Gateway 2.0 URLs. Logs are written to stderr. `GET /account` returns the same resolved topology. +Automation must consume these full URLs rather than reconstructing them from the requested ports. + +```json +{ + "event": "ready", + "managementEndpoint": "http://127.0.0.1:49150/", + "accountEndpoint": "http://127.0.0.1:49151/", + "regions": [ + { + "name": "East US", + "gatewayEndpoint": "http://127.0.0.1:49151/", + "gateway20Endpoint": "http://127.0.0.1:49152/" + } + ] +} +``` + --- ## 5. Data-Plane Hosting (Gateway V1 & Gateway 2.0) @@ -221,10 +240,10 @@ account read) are served here unchanged. ### 5.2 Gateway 2.0 (RNTBD over HTTP/2) -Enabled per region by setting `thinClientPort`. When enabled: +Enabled per region by setting `gateway20Port`. When enabled: 1. The account topology advertises `thinClientReadableLocations` / `thinClientWritableLocations` - pointing at `http://127.0.0.1:{thinClientPort}`. + pointing at the bound `gateway20Endpoint` reported by the host. 2. The Gateway 2.0 listener answers `POST /connectivity-probe` with `200 OK`. 3. Data-plane requests arrive as RNTBD frames wrapped in HTTP/2 POSTs. The listener **decodes** the RNTBD request frame + literal `thinclient` headers, reconstructs the logical operation, dispatches @@ -245,8 +264,9 @@ The management API exposes only the control-plane actions that have **no Cosmos equivalent**. Database, container, offer, and item lifecycle are **not** here — those are served by the standard Cosmos data-plane endpoints on the region gateway ports (or seeded via config). -All endpoints below are served on the management port (e.g. `:9090`), use JSON bodies, and -return JSON. Errors use conventional HTTP status codes with a `{ "error": "..." }` body. +All endpoints below are served on the resolved `managementEndpoint` from the startup `ready` +record, use JSON bodies, and return JSON. Errors use conventional HTTP status codes with a +`{ "error": "..." }` body. ### 6.1 Introspection @@ -274,19 +294,31 @@ POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split body (optional): { "mode": "midpoint" | "epk" | "storage", // default: "midpoint" "epk": "", // required when mode = "epk"; ignored otherwise - "lockDurationMs": 0 // optional; 410/1007 window before children appear + "lockMode": "instant" | "delayed" | "manual", // default: "instant" + "lockDurationMs": 500 // required only when lockMode = "delayed" } - → 202 { "operationId": "op-split-123", "status": "Running" } + → 202 { "operationId": "op-split-123", "status": "Running", "phase": "Preparing" } POST /databases/{db}/containers/{coll}/partitions/merge - body: { "partitionIds": [4, 5] } // exactly two adjacent partitions - → 202 { "operationId": "op-merge-456", "status": "Running" } + body: { + "partitionIds": [4, 5], // exactly two adjacent partitions + "lockMode": "manual" + } + → 202 { "operationId": "op-merge-456", "status": "Running", "phase": "Preparing" } GET /operations/{operationId} - → 200 { "operationId": "...", "status": "Running" | "Succeeded" | "Failed" } + → 200 { + "operationId": "...", + "status": "Running" | "Succeeded" | "Failed", + "phase": "Preparing" | "Swapping" | "Succeeded" | "Failed" + } // On success, include operation-specific result details: // split: { "database": "testdb", "container": "testcoll", "parent": 0, "children": [4, 5], "mode": "storage", "splitEpk": "6A3C000000000000000000000000000000" } // merge: { "merged": [4, 5], "into": 6 } + +POST /operations/{operationId}/advance + → 200 { "operationId": "...", "status": "Running", "phase": "Swapping" } + // Valid only for manual operations. Each call advances one phase. ``` The operation result (via `GET /operations/{operationId}`) echoes the resolved `mode` and the @@ -294,26 +326,48 @@ concrete `splitEpk` that was applied, so a caller that requested `midpoint` or ` the boundary the emulator chose. Only the `epk` hex parser and the `storage` mode are new code behind this endpoint; `midpoint` and the `epk` split execution reuse existing store hooks. +Operation phase semantics are deterministic: + +- `Preparing`: source partitions remain available and replacement partitions are hidden. +- `Swapping`: source partition requests return `410/1007` and replacements remain hidden. +- `Succeeded`: source partitions are removed and replacements are available. +- `Failed`: no further advancement is allowed; the operation contains the error. + +`instant` operations advance automatically without an observable lock-window guarantee. `delayed` +operations hold `Swapping` for the configured `lockDurationMs`. `manual` operations advance only +through `POST /operations/{operationId}/advance`, allowing reliable assertions in every phase. + Samples — one request per split mode: ```bash +# Read this URL from the host's JSON ready record. +management_endpoint="http://127.0.0.1:49150/" + # 1) Mid-point split (default): halve the partition's EPK range geometrically. -curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split +curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" # 2) Custom-EPK split: split at an explicit hex EPK boundary. -curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ +curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ -H 'content-type: application/json' \ -d '{ "mode": "epk", "epk": "3FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" }' # 3) Storage-based split: pick a boundary that balances documents across the children. -curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ +curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ -H 'content-type: application/json' \ -d '{ "mode": "storage" }' -# Any mode may set a non-zero lock window; the partition returns 410/1007 until it elapses. -curl -X POST http://127.0.0.1:9090/databases/testdb/containers/testcoll/partitions/0/split \ +# Delayed mode provides a timed 410/1007 window. +curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ + -H 'content-type: application/json' \ + -d '{ "mode": "midpoint", "lockMode": "delayed", "lockDurationMs": 500 }' + +# Manual mode allows deterministic phase-by-phase assertions. +operation_id=$(curl -sS -X POST \ + "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ -H 'content-type: application/json' \ - -d '{ "mode": "midpoint", "lockDurationMs": 500 }' + -d '{ "mode": "midpoint", "lockMode": "manual" }' | jq -r .operationId) +curl -X POST "${management_endpoint}operations/${operation_id}/advance" # Preparing → Swapping +curl -X POST "${management_endpoint}operations/${operation_id}/advance" # Swapping → Succeeded ``` ### 6.3 Region offline / online (PR2) @@ -332,7 +386,7 @@ An offline region is dropped from the account topology's readable/writable locat Sample: ```bash -curl -X POST http://127.0.0.1:9090/regions/West%20US/offline +curl -X POST "${management_endpoint}regions/West%20US/offline" ``` ### 6.4 Runtime write-region failover (PR2) @@ -349,7 +403,7 @@ reflects the new `writableLocations`, and the driver re-routes writes accordingl Sample: ```bash -curl -X POST http://127.0.0.1:9090/failover \ +curl -X POST "${management_endpoint}failover" \ -H 'content-type: application/json' \ -d '{ "writeRegion": "West US" }' ``` @@ -380,7 +434,7 @@ taking it fully offline. ## 7. Rust Public API Surface -This is the surface the host crate consumes, all behind `__internal_in_memory_emulator_host`. +This is the surface the host crate consumes, all behind `__internal_in_memory_emulator`. Items marked **(PR2)** are new primitives introduced in the second PR and are shown here as the proposed signatures. @@ -498,8 +552,9 @@ matcher is a **contingency** only if end-to-end validation reveals a gap (ADR-00 - Add `test_category = "emulator_inmemory"` to the `#[cfg_attr(...)]` gates and ignore messages of the existing emulator suites in both crates, preserving intentional legacy-only exclusions. - Extend `sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` to build and start the host binary - with a provisioning config, wait for `GET /health`, then set `AZURE_COSMOS_CONNECTION_STRING` - to the hosted endpoint and the `emulator_inmemory` cfg. + with a provisioning config, parse the JSON `ready` record from stdout, wait for `GET /health` on + the resolved management endpoint, then build `AZURE_COSMOS_CONNECTION_STRING` from the reported + `accountEndpoint` and set the `emulator_inmemory` cfg. - Add a `ContinueOnError` matrix leg to `sdk/cosmos/ci.yml` modeled on `Cosmos_vnext_emulator`, running the **existing** emulator suites against the hosted emulator in **both** Gateway V1 and Gateway 2.0 modes. This is the real end-to-end validation of h2c + RNTBD. From 6332052df95e41c561ff041d1dbee3a96ff3b6d9 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:43:44 +0000 Subject: [PATCH 26/44] Update 007_gateway_v2_rntbd.md --- .../docs/adr/007_gateway_v2_rntbd.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md index 6e569070189..c3b4c865fda 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/007_gateway_v2_rntbd.md @@ -20,15 +20,15 @@ Keep request decoding and response encoding alongside the existing RNTBD client an RNTBD-framed request, dispatches it through the existing in-memory operation handlers, and returns an RNTBD-framed response. Token and frame internals remain private to the driver. -Gateway 2.0 is enabled per region by configuring a thin-client endpoint. Only then does account +Gateway 2.0 is enabled per region by configuring a Gateway 2.0 endpoint. Only then does account discovery advertise `thinClientReadableLocations` and `thinClientWritableLocations`. The -thin-client listener answers `POST /connectivity-probe` and requires HTTP/2. Unsupported RNTBD +Gateway 2.0 listener answers `POST /connectivity-probe` and requires HTTP/2. Unsupported RNTBD semantics are rejected explicitly instead of being silently discarded. ## Consequences The driver remains the single owner of RNTBD wire compatibility. The host depends on a small -unstable adapter rather than public token types. Accounts without thin-client endpoints retain +unstable adapter rather than public token types. Accounts without Gateway 2.0 endpoints retain standard gateway behavior, while configured regions exercise the same discovery and probe flow as the service. From d9e1ad8ce139968e56f8e84ca3cf436677b0eebc Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:46:50 +0000 Subject: [PATCH 27/44] Addressing code review feedback --- .../docs/adr/003_port_per_region.md | 9 +++++---- .../docs/adr/005_management_rest_api.md | 16 ++++++++-------- .../azure_data_cosmos_emulator/docs/plan.md | 16 ++++++++-------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md index 41138dfe632..0858ddaa8fc 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/003_port_per_region.md @@ -17,10 +17,11 @@ Bind one gateway listener per region to a distinct loopback endpoint. A single s single-region account has one standard gateway endpoint. Gateway 2.0 adds an optional second RNTBD endpoint per region. -Every configured port is optional and defaults to `0`, which asks the operating system to assign -an available port. An explicit non-zero port remains available for interactive scenarios that -need a stable endpoint. Account discovery advertises the **actual bound URLs**, never the requested -port value. +The standard gateway and management ports default to `0`, which asks the operating system to assign +available ports. Gateway 2.0 is enabled by including `gateway20Port`; its value may also be `0` for +OS assignment, while omission disables that optional listener. Explicit non-zero ports remain +available for interactive scenarios that need stable endpoints. Account discovery advertises the +**actual bound URLs**, never the requested port values. After all listeners are bound, the host writes one machine-readable JSON `ready` record to stdout. Diagnostic logs go to stderr so automation can parse stdout without log filtering. The record diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md index 87df922c850..fcdbc30896f 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/005_management_rest_api.md @@ -27,18 +27,18 @@ these phases: 3. `Succeeded`: source partitions are gone and replacement partitions are visible. 4. `Failed`: a terminal error is available on the operation resource. -Each operation selects one lock/progression mode: +Each operation selects one progression mode: -- `instant` (default): progress automatically through all phases without a guaranteed observable - `Swapping` window. -- `delayed`: enter `Swapping` automatically, remain there for `lockDurationMs`, then complete. +- `automatic` (default): enter `Swapping` automatically, remain there for `lockDurationMs`, then + complete. The duration defaults to `0`, which provides no guaranteed observable `Swapping` + window. - `manual`: remain in each non-terminal phase until `POST /operations/{operationId}/advance` moves the operation forward by one phase. -`lockDurationMs` is required and positive for `delayed`; it is rejected for `instant` and `manual`. -Advancing an automatic or terminal operation returns `409 Conflict`. The operation state machine, -not a generic lock API, owns partition locking so tests cannot create a lock state unrelated to an -actual topology transition. +`lockDurationMs` must be non-negative and is accepted only for `automatic` operations. Supplying it +for `manual` progression returns `400 Bad Request`. Advancing an automatic or terminal operation +returns `409 Conflict`. The operation state machine, not a generic lock API, owns partition locking +so tests cannot create a lock state unrelated to an actual topology transition. ## Consequences diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 31f76385ca5..8f6f1c96699 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -294,15 +294,15 @@ POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split body (optional): { "mode": "midpoint" | "epk" | "storage", // default: "midpoint" "epk": "", // required when mode = "epk"; ignored otherwise - "lockMode": "instant" | "delayed" | "manual", // default: "instant" - "lockDurationMs": 500 // required only when lockMode = "delayed" + "progressionMode": "automatic" | "manual", // default: "automatic" + "lockDurationMs": 500 // automatic only; default: 0 } → 202 { "operationId": "op-split-123", "status": "Running", "phase": "Preparing" } POST /databases/{db}/containers/{coll}/partitions/merge body: { "partitionIds": [4, 5], // exactly two adjacent partitions - "lockMode": "manual" + "progressionMode": "manual" } → 202 { "operationId": "op-merge-456", "status": "Running", "phase": "Preparing" } @@ -333,8 +333,8 @@ Operation phase semantics are deterministic: - `Succeeded`: source partitions are removed and replacements are available. - `Failed`: no further advancement is allowed; the operation contains the error. -`instant` operations advance automatically without an observable lock-window guarantee. `delayed` -operations hold `Swapping` for the configured `lockDurationMs`. `manual` operations advance only +`automatic` operations hold `Swapping` for the configured `lockDurationMs`; the default `0` gives +no observable lock-window guarantee. `manual` operations reject `lockDurationMs` and advance only through `POST /operations/{operationId}/advance`, allowing reliable assertions in every phase. Samples — one request per split mode: @@ -356,16 +356,16 @@ curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partiti -H 'content-type: application/json' \ -d '{ "mode": "storage" }' -# Delayed mode provides a timed 410/1007 window. +# Automatic progression can provide a timed 410/1007 window. curl -X POST "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ -H 'content-type: application/json' \ - -d '{ "mode": "midpoint", "lockMode": "delayed", "lockDurationMs": 500 }' + -d '{ "mode": "midpoint", "progressionMode": "automatic", "lockDurationMs": 500 }' # Manual mode allows deterministic phase-by-phase assertions. operation_id=$(curl -sS -X POST \ "${management_endpoint}databases/testdb/containers/testcoll/partitions/0/split" \ -H 'content-type: application/json' \ - -d '{ "mode": "midpoint", "lockMode": "manual" }' | jq -r .operationId) + -d '{ "mode": "midpoint", "progressionMode": "manual" }' | jq -r .operationId) curl -X POST "${management_endpoint}operations/${operation_id}/advance" # Preparing → Swapping curl -X POST "${management_endpoint}operations/${operation_id}/advance" # Swapping → Succeeded ``` From 72165723dcdfc5ca76e0ab1d4c3a20cd79bbfacc Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:47:16 +0000 Subject: [PATCH 28/44] Update 002_separate_host_crate.md --- .../docs/adr/002_separate_host_crate.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md index 045ab19a3b3..21eb2a801db 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/002_separate_host_crate.md @@ -25,9 +25,8 @@ non-SemVer (the `__internal_` prefix), so stable builds are unaffected. ## Alternatives +- Extracting the emulator into its own library crate was rejected because it would force a large, unstable slice of driver internals to become public. - internal feature already communicates the same stability boundary. A second feature would add - feature-matrix complexity without providing a meaningful compatibility guarantee. - Adding a second host-specific feature was rejected because the existing emulator feature already owns this internal surface and the split would add no useful granularity. From a168d02f2cff321e1fce4d9181eaea8bc98cdfc6 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 20:48:35 +0000 Subject: [PATCH 29/44] Iterating on doc fixes --- .../adr/009_transport_security_and_authentication.md | 10 +++++----- sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md index e71f168b552..4303b48d860 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/009_transport_security_and_authentication.md @@ -22,11 +22,11 @@ It supports three authentication modes: Entra validation requires all of the following trust inputs: -| Input | Purpose | -| --- | --- | -| JWKS URI or local JWKS file | Verify the token signature. | -| Expected issuer | Restrict the accepted token authority. | -| Expected audience | Prevent tokens issued for another resource from being replayed. | +| Input | Purpose | +| --------------------------------- | ----------------------------------------------------------------- | +| JWKS URI or local JWKS file | Verify the token signature. | +| Expected issuer | Restrict the accepted token authority. | +| Expected audience | Prevent tokens issued for another resource from being replayed. | | Allowed object or application IDs | Restrict authenticated principals after cryptographic validation. | Selecting an authenticated mode without HTTPS, or omitting required trust inputs, is a startup diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 8f6f1c96699..4202bd21af5 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -58,11 +58,11 @@ emulator implementation out of the driver crate (it depends heavily on driver-in The work ships across three pull requests. -| PR | Contents | Feature/cfg | -| ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | -| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator`, `test_category = "emulator_inmemory"` | -| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | -| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | +| PR | Contents | Feature/cfg | +| ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| **PR1** | New host crate; per-region h2c hosting of Gateway V1 **and** Gateway 2.0 (config-gated); management REST API for control-plane actions; CI running the existing emulator suites against the hosted emulator in both gateway modes. | `__internal_in_memory_emulator`, `test_category = "emulator_inmemory"` | +| **PR2** | New emulator store primitives: region offline/online and runtime write-region failover, with net-new tests and their management REST endpoints. | same host feature | +| **PR3** | Optional HTTPS and authentication (primary key, primary read-only key, Entra ID with an allow-list of object IDs). | new `auth` config block | ### Feature gating From bb83bd216f3fd3973f8a09a4e3f229c68035f2d4 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 15 Jul 2026 21:54:02 +0000 Subject: [PATCH 30/44] Initial implementation --- Cargo.lock | 933 +++++++-------- Cargo.toml | 2 + sdk/cosmos/azure_data_cosmos/build.rs | 2 +- .../tests/emulator_tests/cosmos_aad.rs | 24 +- .../tests/emulator_tests/cosmos_batch.rs | 40 +- .../emulator_tests/cosmos_change_feed.rs | 80 +- .../tests/emulator_tests/cosmos_containers.rs | 20 +- .../tests/emulator_tests/cosmos_databases.rs | 8 +- .../emulator_tests/cosmos_fault_injection.rs | 96 +- .../emulator_tests/cosmos_feed_ranges.rs | 48 +- .../tests/emulator_tests/cosmos_items.rs | 100 +- .../tests/emulator_tests/cosmos_offers.rs | 20 +- .../tests/emulator_tests/cosmos_patch.rs | 40 +- .../tests/emulator_tests/cosmos_proxy.rs | 4 + .../tests/emulator_tests/cosmos_query.rs | 76 +- .../cosmos_response_metadata.rs | 24 +- sdk/cosmos/azure_data_cosmos_driver/build.rs | 2 +- .../src/driver/routing/routing_systems.rs | 8 +- .../driver/transport/gateway_v2_dispatch.rs | 28 + .../src/driver/transport/rntbd/request.rs | 124 +- .../src/driver/transport/rntbd/response.rs | 125 ++ .../src/driver/transport/rntbd/tokens.rs | 23 +- .../src/in_memory_emulator/config.rs | 55 +- .../src/in_memory_emulator/gateway_v2.rs | 557 +++++++++ .../src/in_memory_emulator/mod.rs | 4 +- .../src/in_memory_emulator/operations.rs | 46 +- .../src/in_memory_emulator/response.rs | 1 + .../src/in_memory_emulator/store.rs | 470 +++++++- .../in_memory_emulator/system_properties.rs | 50 +- .../emulator_tests/driver_fault_injection.rs | 56 +- .../driver_hedging_kill_switch.rs | 8 +- .../emulator_tests/driver_item_operations.rs | 24 +- .../tests/emulator_tests/driver_patch.rs | 56 +- .../emulator_tests/hosted_emulator_ci.rs | 60 + .../tests/emulator_tests/mod.rs | 1 + .../in_memory_emulator_tests/error_cases.rs | 23 + .../azure_data_cosmos_emulator/Cargo.toml | 34 + .../config/ci-gateway-v1.json | 25 + .../config/ci-gateway-v2.json | 26 + .../azure_data_cosmos_emulator/docs/plan.md | 12 +- .../azure_data_cosmos_emulator/src/config.rs | 520 +++++++++ .../src/data_plane.rs | 150 +++ .../src/gateway_v2.rs | 272 +++++ .../azure_data_cosmos_emulator/src/main.rs | 154 +++ .../src/management.rs | 1004 +++++++++++++++++ .../azure_data_cosmos_emulator/src/metrics.rs | 28 + sdk/cosmos/ci.yml | 6 + .../eng/scripts/Invoke-CosmosTestCleanup.ps1 | 34 +- .../eng/scripts/Invoke-CosmosTestSetup.ps1 | 125 ++ sdk/cosmos/inmemory-emulator-matrix.json | 17 + 50 files changed, 4827 insertions(+), 818 deletions(-) create mode 100644 sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs create mode 100644 sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/config.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/main.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/management.rs create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/src/metrics.rs create mode 100644 sdk/cosmos/inmemory-emulator-matrix.json diff --git a/Cargo.lock b/Cargo.lock index 8c204080999..e615593c344 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,18 +110,18 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "async-compression" @@ -165,7 +165,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -176,7 +176,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -187,15 +187,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -203,14 +203,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -220,14 +221,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "bytes", "futures-util", "http", "http-body", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -240,6 +241,40 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "axum-macros", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "axum-core" version = "0.4.5" @@ -260,6 +295,36 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "azure_canary" version = "0.1.0" @@ -396,7 +461,7 @@ checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "tracing", ] @@ -406,7 +471,7 @@ version = "1.1.0-beta.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "tokio", "tracing", "tracing-subscriber", @@ -453,7 +518,7 @@ dependencies = [ "flate2", "futures", "include-file", - "rand 0.10.1", + "rand 0.10.2", "rand_chacha 0.10.0", "reqwest", "serde", @@ -473,7 +538,7 @@ dependencies = [ "azure_core_test", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "tokio", ] @@ -533,7 +598,7 @@ dependencies = [ "h2", "percent-encoding", "quick-xml", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "serde", "serde_json", @@ -559,13 +624,29 @@ dependencies = [ "url", ] +[[package]] +name = "azure_data_cosmos_emulator" +version = "0.1.0" +dependencies = [ + "axum 0.8.9", + "azure_core 1.1.0", + "azure_data_cosmos_driver", + "clap", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "azure_data_cosmos_macros" version = "0.3.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "tracing", ] @@ -583,7 +664,7 @@ dependencies = [ "futures", "hdrhistogram", "hostname", - "rand 0.10.1", + "rand 0.10.2", "serde", "serde_json", "sysinfo", @@ -654,7 +735,7 @@ dependencies = [ "futures", "hmac", "percent-encoding", - "rand 0.10.1", + "rand 0.10.2", "rand_chacha 0.10.0", "rustc_version", "sha2", @@ -699,7 +780,7 @@ dependencies = [ "azure_core_test", "azure_identity 1.0.0", "futures", - "rand 0.10.1", + "rand 0.10.2", "rand_chacha 0.10.0", "serde", "serde_json", @@ -723,7 +804,7 @@ dependencies = [ "futures", "include-file", "openssl", - "rand 0.10.1", + "rand 0.10.2", "rustc_version", "serde", "serde_json", @@ -745,7 +826,7 @@ dependencies = [ "criterion", "futures", "include-file", - "rand 0.10.1", + "rand 0.10.2", "rand_chacha 0.10.0", "rustc_version", "serde", @@ -768,7 +849,7 @@ dependencies = [ "clap", "futures", "include-file", - "rand 0.10.1", + "rand 0.10.2", "rustc_version", "serde", "serde_json", @@ -802,7 +883,7 @@ dependencies = [ "futures", "percent-encoding", "pin-project", - "rand 0.10.1", + "rand 0.10.2", "serde", "serde_json", "time", @@ -846,7 +927,7 @@ dependencies = [ "azure_storage_common", "azure_storage_sas", "futures", - "rand 0.10.1", + "rand 0.10.2", "serde", "time", "tokio", @@ -901,15 +982,15 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -928,9 +1009,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" dependencies = [ "borsh-derive", "bytes", @@ -939,22 +1020,22 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecheck" @@ -986,9 +1067,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cast" @@ -1010,16 +1091,16 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.117", + "syn 2.0.119", "tempfile", "toml", ] [[package]] name = "cc" -version = "1.2.62" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -1041,9 +1122,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -1052,9 +1133,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "num-traits", ] @@ -1088,9 +1169,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" dependencies = [ "clap_builder", "clap_derive", @@ -1098,9 +1179,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1117,7 +1198,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1306,18 +1387,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1334,9 +1415,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1375,7 +1456,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1386,14 +1467,14 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "der" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "pem-rfc7468", "zeroize", @@ -1405,7 +1486,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1422,13 +1502,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1451,9 +1531,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "equivalent" @@ -1605,12 +1685,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foreign-types" version = "0.3.2" @@ -1703,7 +1777,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1774,16 +1848,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", + "wasm-bindgen", ] [[package]] @@ -1804,9 +1878,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1841,15 +1915,6 @@ dependencies = [ "ahash", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1898,9 +1963,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1908,9 +1973,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1918,9 +1983,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1943,15 +2008,15 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -2030,7 +2095,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -2118,12 +2183,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -2159,7 +2218,7 @@ checksum = "babf2fc9613fcd8bf298719ad975819fc6ab212d3a60de4d2eb55554c7bfd03d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2247,7 +2306,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2266,28 +2325,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2319,12 +2377,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" version = "0.2.186" @@ -2354,9 +2406,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -2379,11 +2431,17 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -2409,9 +2467,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -2507,9 +2565,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.80" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ "bitflags", "cfg-if", @@ -2527,7 +2585,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2538,9 +2596,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.116" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -2610,7 +2668,7 @@ dependencies = [ "futures-util", "opentelemetry 0.31.0", "percent-encoding", - "rand 0.9.4", + "rand 0.9.5", "thiserror", ] @@ -2626,7 +2684,7 @@ dependencies = [ "opentelemetry 0.32.0", "percent-encoding", "portable-atomic", - "rand 0.9.4", + "rand 0.9.5", "thiserror", "tokio", ] @@ -2638,7 +2696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", - "rand 0.8.6", + "rand 0.8.7", "serde", ] @@ -2713,7 +2771,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2792,16 +2850,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -2840,7 +2888,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2884,9 +2932,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -2895,7 +2943,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.6.5", "thiserror", "tokio", "tracing", @@ -2904,15 +2952,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -2926,23 +2975,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -2967,9 +3016,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2979,9 +3028,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2989,12 +3038,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -3053,6 +3102,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -3084,9 +3142,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -3096,9 +3154,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -3107,9 +3165,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rend" @@ -3122,9 +3180,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -3207,15 +3265,15 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.42.0" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ "arrayvec", "borsh", "bytes", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "rkyv", "serde", "serde_json", @@ -3224,15 +3282,15 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -3258,9 +3316,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", @@ -3272,9 +3330,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3284,9 +3342,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -3333,9 +3391,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -3346,15 +3410,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scc" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" -dependencies = [ - "sdd", -] - [[package]] name = "schannel" version = "0.1.29" @@ -3370,12 +3425,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sdd" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" - [[package]] name = "seahash" version = "4.1.0" @@ -3423,9 +3472,9 @@ dependencies = [ [[package]] name = "serde_amqp" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e76738e7a058df01b5b33194359930a1aa5bc233dc07e510f458aca47189aea2" +checksum = "ffc615f24778bb6d92510fe82afc74f99fb03e6ddbfd75927356fe681c1b6037" dependencies = [ "bytes", "indexmap 2.14.0", @@ -3447,7 +3496,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3477,14 +3526,14 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3493,6 +3542,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -3501,7 +3561,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3522,30 +3582,41 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serial_test" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" dependencies = [ "futures-executor", "futures-util", "log", "once_cell", "parking_lot", - "scc", "serial_test_derive", ] [[package]] name = "serial_test_derive" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3570,9 +3641,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -3586,15 +3657,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "simd_cesu8" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ "rustc_version", "simdutf8", @@ -3614,9 +3685,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" @@ -3630,9 +3701,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3669,9 +3740,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -3695,7 +3766,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3735,7 +3806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3758,26 +3829,25 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.47" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3787,15 +3857,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -3823,9 +3893,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -3847,7 +3917,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2 0.6.5", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -3861,7 +3931,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3955,14 +4025,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] @@ -3971,14 +4041,14 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] name = "toml_writer" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tonic" @@ -3988,7 +4058,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.7.9", "base64 0.22.1", "bytes", "h2", @@ -4021,7 +4091,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.6", + "rand 0.8.7", "slab", "tokio", "tokio-util", @@ -4043,6 +4113,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -4086,6 +4157,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4099,7 +4171,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4149,7 +4221,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4166,9 +4238,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typespec" @@ -4211,7 +4283,7 @@ dependencies = [ "dyn-clone", "futures", "pin-project", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "rust_decimal", "serde", @@ -4236,7 +4308,7 @@ dependencies = [ "futures", "gloo-timers", "pin-project", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "rust_decimal", "serde", @@ -4260,7 +4332,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4272,7 +4344,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "syn 2.0.117", + "syn 2.0.119", "tokio", "typespec_client_core 1.1.0", ] @@ -4285,15 +4357,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" - -[[package]] -name = "unicode-xid" -version = "0.2.6" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "untrusted" @@ -4363,24 +4429,24 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", - "rand 0.10.1", + "rand 0.10.2", "uuid-rng-internal", "wasm-bindgen", ] [[package]] name = "uuid-rng-internal" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1409d2323564d9b8d01192a77a0604a46bf29eb42e9b535aca34dec65482652b" +checksum = "9dff7387287837acf196f2596a6216f94c555d947d5331e1cc48393131020119" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", ] [[package]] @@ -4428,27 +4494,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -4460,9 +4517,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4470,9 +4527,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4480,48 +4537,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -4535,18 +4570,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "wasmtimer" version = "0.4.3" @@ -4563,9 +4586,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -4583,9 +4606,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -4628,7 +4651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ "windows-core 0.57.0", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4661,7 +4684,7 @@ dependencies = [ "windows-implement 0.57.0", "windows-interface 0.57.0", "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4696,7 +4719,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4707,7 +4730,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4718,7 +4741,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4729,7 +4752,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4754,7 +4777,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4781,16 +4804,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4808,31 +4822,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4850,96 +4847,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" @@ -4948,107 +4897,19 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" @@ -5066,9 +4927,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -5083,28 +4944,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5124,15 +4985,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" @@ -5164,7 +5025,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5183,15 +5044,15 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index e6168a96881..da22b4e3187 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "sdk/cosmos/azure_data_cosmos", "sdk/cosmos/azure_data_cosmos_benchmarks", "sdk/cosmos/azure_data_cosmos_driver", + "sdk/cosmos/azure_data_cosmos_emulator", "sdk/cosmos/azure_data_cosmos_driver_native", "sdk/cosmos/azure_data_cosmos_macros", "sdk/cosmos/azure_data_cosmos_perf", @@ -89,6 +90,7 @@ version = "1.0.0" async-lock = "3.4" async-stream = { version = "0.3.6" } async-trait = "0.1" +axum = { version = "0.8", features = ["http2", "json", "macros"] } base64 = "0.22" arc-swap = "1.7" backtrace = "0.3" diff --git a/sdk/cosmos/azure_data_cosmos/build.rs b/sdk/cosmos/azure_data_cosmos/build.rs index a75eef56071..758eb1dec9d 100644 --- a/sdk/cosmos/azure_data_cosmos/build.rs +++ b/sdk/cosmos/azure_data_cosmos/build.rs @@ -7,6 +7,6 @@ fn main() { // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. println!( - "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"multi_write\", \"split\", \"gateway_v2\", \"gateway_v2_multi_region\"))" + "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"multi_write\", \"split\", \"gateway_v2\", \"gateway_v2_multi_region\"))" ); } diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs index b3cea941ebd..bede8fd05da 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_aad.rs @@ -48,8 +48,16 @@ struct AadTestItem { /// invoked for the Cosmos scope, guarding against silently exercising key auth. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" +)] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator authentication is deferred to PR3" )] pub async fn aad_item_crud_roundtrip() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -149,8 +157,16 @@ pub async fn aad_item_crud_roundtrip() -> Result<(), Box> { /// the `readMetadata` data action the SDK requires on its first request. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" +)] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator authentication is deferred to PR3" )] pub async fn aad_read_container_metadata() -> Result<(), Box> { TestClient::run_with_unique_db( diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_batch.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_batch.rs index 12b8f5d1108..24141d52459 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_batch.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_batch.rs @@ -43,8 +43,12 @@ async fn create_container( #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn batch_create_and_read() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -108,8 +112,12 @@ pub async fn batch_create_and_read() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn batch_mixed_operations() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -189,8 +197,12 @@ pub async fn batch_mixed_operations() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn batch_atomicity_on_failure() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -249,8 +261,12 @@ pub async fn batch_atomicity_on_failure() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn batch_fails_when_exceeding_max_operations() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -295,8 +311,12 @@ pub async fn batch_fails_when_exceeding_max_operations() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -137,8 +145,16 @@ pub async fn change_feed_from_beginning_single_partition() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -179,8 +195,16 @@ pub async fn change_feed_from_beginning_full_container() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -242,8 +266,12 @@ pub async fn change_feed_start_from_now_returns_only_new_changes() -> Result<(), /// without erroring or terminating the stream. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn change_feed_no_changes_returns_empty_page() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -286,8 +314,16 @@ pub async fn change_feed_no_changes_returns_empty_page() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -458,8 +494,16 @@ pub async fn change_feed_now_resume_does_not_replay_history() -> Result<(), Box< /// to keep the boundary unambiguous. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" +)] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator does not yet model change-feed start and continuation semantics" )] pub async fn change_feed_point_in_time_excludes_earlier_changes() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -519,8 +563,16 @@ pub async fn change_feed_point_in_time_excludes_earlier_changes() -> Result<(), /// still surfacing every item exactly once. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" +)] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator does not yet model change-feed start and continuation semantics" )] pub async fn change_feed_max_item_count_pages_backlog() -> Result<(), Box> { const PAGE_LIMIT: u32 = 10; diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_containers.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_containers.rs index 8e57394e19a..701d345b902 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_containers.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_containers.rs @@ -20,13 +20,21 @@ use framework::{TestClient, TestOptions}; #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", ignore = "skipped on vnext emulator: behavioral divergence" )] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator does not yet support replacing container properties" +)] pub async fn container_crud_simple() -> Result<(), Box> { TestClient::run_with_unique_db( async |run_context, db_client| { @@ -148,8 +156,12 @@ pub async fn container_crud_simple() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_databases.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_databases.rs index 5fffb4fb412..897ed01fc07 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_databases.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_databases.rs @@ -12,8 +12,12 @@ use futures::TryStreamExt; #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn database_crud() -> Result<(), Box> { TestClient::run_with_options( diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_fault_injection.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_fault_injection.rs index 0277f3cf3e0..143455bdc18 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_fault_injection.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_fault_injection.rs @@ -49,8 +49,12 @@ fn create_test_item(unique_id: &str) -> TestItem { /// With probability 0.0, the fault should never be applied. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_probability_zero_never_fails() -> Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -115,8 +119,12 @@ pub async fn fault_injection_probability_zero_never_fails() -> Result<(), Box Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -184,8 +192,12 @@ pub async fn fault_injection_probability_one_always_fails() -> Result<(), Box Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -251,8 +263,12 @@ pub async fn fault_injection_429_retry_with_hit_limit() -> Result<(), Box Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -337,8 +353,12 @@ pub async fn fault_injection_delete_item_fault_crud_succeeds() -> Result<(), Box /// Test container-specific fault - verify other containers unaffected. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_container_specific() -> Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -429,8 +449,12 @@ pub async fn fault_injection_container_specific() -> Result<(), Box> /// Test multiple rules priority - first matching rule wins. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_multiple_rules_priority() -> Result<(), Box> { // First rule: 429 for ReadItem @@ -504,8 +528,12 @@ pub async fn fault_injection_multiple_rules_priority() -> Result<(), Box Result<(), Box> { // First rule: 429 for ReadItem, but with a future start_time (won't be active yet) @@ -580,8 +608,12 @@ pub async fn fault_injection_first_rule_inactive_due_to_start_time() -> Result<( /// Second rule should win. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_first_rule_expired_due_to_end_time() -> Result<(), Box> { // First rule: 429 for ReadItem, but with an end_time in the past (already expired) @@ -658,8 +690,12 @@ pub async fn fault_injection_first_rule_expired_due_to_end_time() -> Result<(), /// Test hit_limit behavior - fault stops after N applications. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_hit_limit_behavior() -> Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -740,8 +776,12 @@ pub async fn fault_injection_hit_limit_behavior() -> Result<(), Box> /// Test empty rules - no fault injection, operations should succeed. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_empty_rules() -> Result<(), Box> { let fault_builder: Vec> = Vec::new(); @@ -791,8 +831,12 @@ pub async fn fault_injection_empty_rules() -> Result<(), Box> { /// Test that item operations succeed when metadata operations are faulted. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_metadata_fault_item_ops_succeed() -> Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() @@ -900,8 +944,12 @@ pub async fn fault_injection_metadata_fault_item_ops_succeed() -> Result<(), Box /// and re-enabling it resumes injection. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_enable_disable_rule() -> Result<(), Box> { let server_error = FaultInjectionResultBuilder::new() diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_feed_ranges.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_feed_ranges.rs index 21f2b85462e..d868dad31c2 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_feed_ranges.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_feed_ranges.rs @@ -22,8 +22,12 @@ use framework::{TestClient, TestOptions}; #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -93,8 +97,12 @@ pub async fn read_feed_ranges_returns_physical_partitions() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -152,8 +160,12 @@ pub async fn feed_range_from_partition_key_maps_correctly() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -190,8 +202,12 @@ pub async fn feed_range_from_full_hpk_returns_single_range() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -258,8 +274,12 @@ pub async fn feed_range_from_prefix_hpk_returns_ranges() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -340,8 +360,12 @@ async fn drain_ids( /// that lets the gateway accept the emitted interior EPK window — they pass. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_items.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_items.rs index d95b2f93e39..12685cbeb3f 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_items.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_items.rs @@ -212,8 +212,12 @@ async fn create_v1_container( #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -441,8 +445,12 @@ pub async fn v1_container_item_crud() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -507,8 +515,12 @@ pub async fn item_read_system_properties() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -564,8 +576,12 @@ pub async fn item_upsert_new() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -628,8 +644,12 @@ pub async fn item_upsert_existing() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -727,8 +747,12 @@ pub async fn item_null_partition_key() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -826,8 +850,12 @@ pub async fn item_replace_if_match_etag() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -925,8 +953,12 @@ pub async fn item_upsert_if_match_etag() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -1042,13 +1074,21 @@ struct ExplicitPkItem { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", ignore = "skipped on vnext emulator: behavioral divergence" )] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator does not yet model the undefined partition-key sentinel" +)] pub async fn item_undefined_partition_key() -> Result<(), Box> { TestClient::run_with_shared_db( async |run_context, _db_client| { @@ -1198,8 +1238,12 @@ pub async fn item_undefined_partition_key() -> Result<(), Box> { /// item already exists. This exercises the driver's error-path bridging. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -1257,8 +1301,12 @@ pub async fn create_item_duplicate_returns_conflict() -> Result<(), Box Result<(), Box> { /// metadata: session token, activity ID, request charge, and server duration. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_offers.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_offers.rs index 1150a8655bd..bf5829863e6 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_offers.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_offers.rs @@ -12,8 +12,12 @@ use framework::{TestClient, TestOptions}; #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -60,13 +64,21 @@ pub async fn container_throughput_crud_manual() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", ignore = "skipped on vnext emulator: behavioral divergence" )] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "hosted in-memory emulator does not yet model autoscale throughput" +)] pub async fn container_throughput_crud_autoscale() -> Result<(), Box> { TestClient::run_with_unique_db( async |run_context, db_client| { diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_patch.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_patch.rs index 484581a9ffe..0d057fe412c 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_patch.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_patch.rs @@ -62,8 +62,12 @@ async fn create_container( /// tests in `azure_data_cosmos_driver::driver::pipeline::patch_handler`. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn patch_item_round_trip() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -140,8 +144,12 @@ pub async fn patch_item_round_trip() -> Result<(), Box> { /// `rmw_propagates_read_error_immediately`. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn patch_item_missing_returns_not_found() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -187,8 +195,12 @@ pub async fn patch_item_missing_returns_not_found() -> Result<(), Box /// underlying loop semantics. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn patch_item_honors_max_attempts_option() -> Result<(), Box> { TestClient::run_with_shared_db( @@ -305,8 +317,12 @@ async fn setup_fault_injected_container( /// Mirrors the driver-level emulator test `cosmos_patch_412_retry`. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn patch_item_412_retry_succeeds() -> Result<(), Box> { let rule = build_replace_412_rule("sdk-patch-412-once", Some(1)); @@ -370,8 +386,12 @@ pub async fn patch_item_412_retry_succeeds() -> Result<(), Box> { /// Mirrors the driver-level emulator test `cosmos_patch_412_exhaustion`. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn patch_item_412_exhaustion_surfaces_precondition_failed() -> Result<(), Box> { diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_proxy.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_proxy.rs index c41768dd447..6ebe955f6be 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_proxy.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_proxy.rs @@ -65,6 +65,10 @@ pub async fn proxy_disabled_by_default_ignores_env() -> Result<(), Box Result<(), Box> { // Skip on the vnext (Linux) emulator pipeline: the vnext gateway does // not honor an outbound proxy in the same way the legacy emulator does diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_query.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_query.rs index ceda74a5bf5..9b8f049a919 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_query.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_query.rs @@ -121,8 +121,12 @@ struct ItemProjection { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn single_partition_query_simple() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -150,8 +154,12 @@ pub async fn single_partition_query_simple() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn single_partition_query_with_parameters() -> Result<(), Box> { TestClient::run_with_unique_db( @@ -189,8 +197,12 @@ pub async fn single_partition_query_with_parameters() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -224,8 +236,12 @@ pub async fn single_partition_query_with_projection() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -256,13 +272,21 @@ pub async fn cross_partition_query_with_projection_and_filter() -> Result<(), Bo #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", ignore = "skipped on vnext emulator: behavioral divergence" )] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "in-memory query planner does not model CrossPartitionQueryNotServable" +)] pub async fn cross_partition_query_with_order_by_fails() -> Result<(), Box> { TestClient::run_with_unique_db( async |_, db_client| { @@ -322,13 +346,21 @@ pub async fn cross_partition_query_with_order_by_fails() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( async |_, db_client| { @@ -417,8 +449,12 @@ pub async fn query_returns_index_and_query_metrics() -> Result<(), Box Result<(), Box> { TestClient::run_with_unique_db( @@ -453,8 +489,16 @@ pub async fn single_partition_query_pagination() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" +)] +#[cfg_attr( + test_category = "emulator_inmemory", + ignore = "unordered cross-partition row order differs in the in-memory emulator" )] pub async fn cross_partition_query_pagination() -> Result<(), Box> { TestClient::run_with_unique_db( diff --git a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_response_metadata.rs b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_response_metadata.rs index e2c75115e25..2f7217607ef 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_response_metadata.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_response_metadata.rs @@ -48,8 +48,12 @@ fn cosmos_headers_from_error(error: &azure_data_cosmos::CosmosError) -> Response #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -99,8 +103,12 @@ pub async fn response_metadata_on_missing_read() -> Result<(), Box> { #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] #[cfg_attr( test_category = "emulator_vnext", @@ -241,8 +249,12 @@ pub async fn response_metadata_on_read_write_preserves_session_and_lsn( #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn query_pages_do_not_leak_lsn_in_items() -> Result<(), Box> { TestClient::run_with_shared_db( diff --git a/sdk/cosmos/azure_data_cosmos_driver/build.rs b/sdk/cosmos/azure_data_cosmos_driver/build.rs index 2fcfa6a3943..5342ab0f052 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/build.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/build.rs @@ -7,6 +7,6 @@ fn main() { // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. println!( - "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"multi_write\", \"multi_region\", \"gateway_v2\", \"gateway_v2_multi_region\", \"native_query_plan\"))" + "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"multi_write\", \"multi_region\", \"gateway_v2\", \"gateway_v2_multi_region\", \"native_query_plan\"))" ); } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/routing/routing_systems.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/routing/routing_systems.rs index 6a7bc44ad88..8cbe3c0afa2 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/routing/routing_systems.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/routing/routing_systems.rs @@ -12,7 +12,7 @@ use std::{ use tracing::warn; -use crate::driver::cache::AccountProperties; +use crate::driver::{cache::AccountProperties, transport::is_emulator_host}; use crate::options::{PartitionFailoverOptions, Region}; use super::{ @@ -118,12 +118,14 @@ fn parse_gateway_v2_locations( for region in gateway_v2_locations { let url = region.database_account_endpoint.url().clone(); - if url.scheme() != "https" { + if url.scheme() != "https" + && !(url.scheme() == "http" && is_emulator_host(®ion.database_account_endpoint)) + { warn!( region = %region.name, endpoint = %region.database_account_endpoint, scheme = url.scheme(), - "Ignoring non-HTTPS Gateway 2.0 endpoint URL" + "Ignoring non-HTTPS Gateway 2.0 endpoint URL outside the emulator" ); continue; } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index a0fed9cee33..2c0ace1be5f 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -407,6 +407,7 @@ pub(crate) fn wrap_request_for_gateway_v2( pub(crate) fn unwrap_response_for_gateway_v2( response: HttpResponse, ) -> azure_core::Result { + let outer_headers = response.headers.clone(); let response = RntbdResponse::read(&response.body)?; let status = u16::from(response.status.status_code()); if !(100..=599).contains(&status) { @@ -460,9 +461,36 @@ pub(crate) fn unwrap_response_for_gateway_v2( if let Some(global_committed_lsn) = response.global_committed_lsn.filter(|value| *value != 0) { headers.insert(X_MS_GLOBAL_COMMITTED_LSN, global_committed_lsn.to_string()); } + if let Some(partition_key_range_id) = response.partition_key_range_id { + headers.insert( + response_header_names::PARTITION_KEY_RANGE_ID, + partition_key_range_id, + ); + } + if let Some(transport_request_id) = response.transport_request_id { + headers.insert( + response_header_names::TRANSPORT_REQUEST_ID, + transport_request_id.to_string(), + ); + } if let Some(owner_full_name) = response.owner_full_name { headers.insert(response_header_names::OWNER_FULL_NAME, owner_full_name); } + for name in [ + response_header_names::SERVER_DURATION_MS, + response_header_names::LSN, + response_header_names::ITEM_LSN, + response_header_names::GLOBAL_COMMITTED_LSN, + response_header_names::QUERY_METRICS, + response_header_names::INDEX_METRICS, + ] { + let header = HeaderName::from_static(name); + if headers.get_optional_str(&header).is_none() { + if let Some(value) = outer_headers.get_optional_str(&header) { + headers.insert(header, value.to_owned()); + } + } + } Ok(HttpResponse { status, diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/request.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/request.rs index c0203146e37..4a8594d5d4b 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/request.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/request.rs @@ -10,6 +10,8 @@ use crate::models::{OperationType, ResourceType}; use super::tokens::{ data_conversion_error, write_uuid_le, RntbdOperationType, RntbdResourceType, Token, }; +#[cfg(any(test, feature = "__internal_in_memory_emulator"))] +use super::tokens::{read_u16_le, read_u32_le, read_uuid_le}; /// A Gateway 2.0 RNTBD request frame. /// @@ -30,6 +32,64 @@ pub(crate) struct RntbdRequestFrame { } impl RntbdRequestFrame { + /// Reads a Gateway 2.0 RNTBD request frame. + #[cfg(any(test, feature = "__internal_in_memory_emulator"))] + pub(crate) fn read(bytes: &[u8]) -> azure_core::Result { + let mut src = bytes; + let header_len = read_u32_le(&mut src)? as usize; + if header_len < 24 { + return Err(data_conversion_error(format!( + "RNTBD request header length {header_len} is smaller than 24 bytes" + ))); + } + if header_len > bytes.len() { + return Err(data_conversion_error(format!( + "RNTBD request header length {header_len} exceeds buffer length {}", + bytes.len() + ))); + } + + let resource_type = + ResourceType::try_from(RntbdResourceType::try_from(read_u16_le(&mut src)?)?)?; + let operation_type = + OperationType::try_from(RntbdOperationType::try_from(read_u16_le(&mut src)?)?)?; + let activity_id = read_uuid_le(&mut src)?; + + let metadata_len = header_len - 24; + let metadata_bytes = src.get(..metadata_len).ok_or_else(|| { + data_conversion_error(format!( + "RNTBD request header length {header_len} exceeds buffer" + )) + })?; + let mut metadata_src = metadata_bytes; + let mut metadata = Vec::new(); + while !metadata_src.is_empty() { + metadata.push(Token::read_from(&mut metadata_src)?); + } + + let mut tail = &bytes[header_len..]; + let body = if tail.is_empty() { + None + } else { + let payload_len = read_u32_le(&mut tail)? as usize; + if tail.len() != payload_len { + return Err(data_conversion_error(format!( + "RNTBD request payload length {payload_len} did not match remaining bytes {}", + tail.len() + ))); + } + Some(tail.to_vec()) + }; + + Ok(Self { + resource_type, + operation_type, + activity_id, + metadata, + body, + }) + } + /// Writes the request frame as Gateway 2.0 RNTBD bytes into `out`. /// /// Streaming into a caller-provided [`std::io::Write`] avoids forcing a @@ -91,10 +151,7 @@ impl RntbdRequestFrame { #[cfg(test)] mod tests { use super::*; - use crate::driver::transport::rntbd::tokens::{ - data_conversion_error, read_u16_le, read_u32_le, read_uuid_le, RntbdOperationType, - RntbdResourceType, TokenValue, - }; + use crate::driver::transport::rntbd::tokens::{RntbdOperationType, TokenValue}; /// Serializes a frame into a fresh `Vec` for assertions. fn serialize(frame: &RntbdRequestFrame) -> azure_core::Result> { @@ -130,7 +187,7 @@ mod tests { }; let bytes = serialize(&frame).unwrap(); - let parsed = parse_request_for_tests(&bytes, frame.body.is_some()).unwrap(); + let parsed = RntbdRequestFrame::read(&bytes).unwrap(); assert_eq!(parsed, frame); } @@ -174,7 +231,7 @@ mod tests { }; let bytes = serialize(&frame).unwrap(); - let parsed = parse_request_for_tests(&bytes, false).unwrap(); + let parsed = RntbdRequestFrame::read(&bytes).unwrap(); assert_eq!(parsed, frame); } @@ -479,59 +536,4 @@ mod tests { "230000000300030078563412ab90efcd0123456789abcdef0400020001000002000001020000007b7d"; assert_eq!(hex, expected); } - - fn parse_request_for_tests( - bytes: &[u8], - has_body: bool, - ) -> azure_core::Result { - let mut src = bytes; - // The leading length field is the request HEADER length (length field - // itself + resource/operation type + activity id + metadata tokens) and - // does NOT include the body length prefix or body bytes. - let header_len = read_u32_le(&mut src)? as usize; - let resource_type = - ResourceType::try_from(RntbdResourceType::try_from(read_u16_le(&mut src)?)?)?; - let operation_type = - OperationType::try_from(RntbdOperationType::try_from(read_u16_le(&mut src)?)?)?; - let activity_id = read_uuid_le(&mut src)?; - - let mut metadata = Vec::new(); - // Bytes consumed so far: 4 (length) + 2 (resource) + 2 (operation) + 16 (activity) = 24. - let metadata_end = header_len.saturating_sub(24); - let metadata_bytes = src.get(..metadata_end).ok_or_else(|| { - data_conversion_error(format!("request header length {header_len} exceeds buffer")) - })?; - let mut metadata_src = metadata_bytes; - while !metadata_src.is_empty() { - metadata.push(Token::read_from(&mut metadata_src)?); - } - src = &src[metadata_end..]; - - let body = if has_body { - let payload_len = read_u32_le(&mut src)? as usize; - if src.len() != payload_len { - return Err(data_conversion_error(format!( - "request payload length {payload_len} did not match remaining bytes {}", - src.len() - ))); - } - Some(src.to_vec()) - } else { - if !src.is_empty() { - return Err(data_conversion_error(format!( - "unexpected {} trailing bytes after header section", - src.len() - ))); - } - None - }; - - Ok(RntbdRequestFrame { - resource_type, - operation_type, - activity_id, - metadata, - body, - }) - } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs index c8c8085ae97..d726f70379c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs @@ -7,6 +7,8 @@ use uuid::Uuid; use crate::models::CosmosStatus; +#[cfg(any(test, feature = "__internal_in_memory_emulator"))] +use super::tokens::{write_uuid_le, TokenValue}; use super::{ status::map_rntbd_status_to_cosmos_status, tokens::{data_conversion_error, read_u32_le, read_uuid_le, RntbdResponseToken, Token}, @@ -49,6 +51,105 @@ pub(crate) struct RntbdResponse { } impl RntbdResponse { + /// Writes this response as a Gateway 2.0 RNTBD frame. + #[cfg(any(test, feature = "__internal_in_memory_emulator"))] + pub(crate) fn write(&self, out: &mut impl std::io::Write) -> azure_core::Result<()> { + let mut metadata = Vec::with_capacity(13); + metadata.push(Token::new( + RntbdResponseToken::PayloadPresent, + TokenValue::Byte(u8::from(!self.body.is_empty())), + )); + if let Some(value) = &self.continuation_token { + metadata.push(Token::new( + RntbdResponseToken::ContinuationToken, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = &self.etag { + metadata.push(Token::new( + RntbdResponseToken::ETag, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = self.retry_after_ms { + metadata.push(Token::new( + RntbdResponseToken::RetryAfterMilliseconds, + TokenValue::ULong(value), + )); + } + if let Some(value) = self.lsn { + metadata.push(Token::new( + RntbdResponseToken::Lsn, + TokenValue::LongLong(value), + )); + } + if let Some(value) = self.request_charge { + metadata.push(Token::new( + RntbdResponseToken::RequestCharge, + TokenValue::Double(value), + )); + } + if let Some(value) = &self.owner_full_name { + metadata.push(Token::new( + RntbdResponseToken::OwnerFullName, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = self.status.sub_status() { + metadata.push(Token::new( + RntbdResponseToken::SubStatus, + TokenValue::ULong(value.value() as u32), + )); + } + if let Some(value) = &self.partition_key_range_id { + metadata.push(Token::new( + RntbdResponseToken::PartitionKeyRangeId, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = self.item_lsn { + metadata.push(Token::new( + RntbdResponseToken::ItemLsn, + TokenValue::LongLong(value), + )); + } + if let Some(value) = self.global_committed_lsn { + metadata.push(Token::new( + RntbdResponseToken::GlobalCommittedLsn, + TokenValue::LongLong(value), + )); + } + if let Some(value) = self.transport_request_id { + metadata.push(Token::new( + RntbdResponseToken::TransportRequestId, + TokenValue::ULong(value), + )); + } + if let Some(value) = &self.session_token { + metadata.push(Token::new( + RntbdResponseToken::SessionToken, + TokenValue::String(value.clone()), + )); + } + + let metadata_len: usize = metadata.iter().map(Token::encoded_len).sum(); + let header_len = u32::try_from(24 + metadata_len) + .map_err(|_| data_conversion_error("RNTBD response header length exceeds u32::MAX"))?; + out.write_all(&header_len.to_le_bytes())?; + out.write_all(&u32::from(u16::from(self.status.status_code())).to_le_bytes())?; + write_uuid_le(out, self.activity_id)?; + for token in metadata { + token.write_to(out)?; + } + if !self.body.is_empty() { + let body_len = u32::try_from(self.body.len()) + .map_err(|_| data_conversion_error("RNTBD response body exceeds u32::MAX"))?; + out.write_all(&body_len.to_le_bytes())?; + out.write_all(&self.body)?; + } + Ok(()) + } + /// Reads a Gateway 2.0 RNTBD response frame. /// /// Wire layout: @@ -207,6 +308,30 @@ mod tests { use crate::driver::transport::rntbd::tokens::{write_uuid_le, TokenValue}; + #[test] + fn response_round_trips_through_server_encoder() { + let response = RntbdResponse { + status: CosmosStatus::new(StatusCode::Created).with_sub_status(1002), + activity_id: Uuid::from_u128(0x1234_5678_90ab_cdef_0123_4567_89ab_cdef), + body: br#"{"id":"doc1"}"#.to_vec(), + continuation_token: Some("next".to_owned()), + etag: Some("etag".to_owned()), + retry_after_ms: Some(10), + lsn: Some(11), + request_charge: Some(5.5), + owner_full_name: Some("dbs/db/colls/coll/docs/doc1".to_owned()), + partition_key_range_id: Some("0".to_owned()), + item_lsn: Some(12), + global_committed_lsn: Some(13), + transport_request_id: Some(14), + session_token: Some("1#12".to_owned()), + }; + let mut bytes = Vec::new(); + response.write(&mut bytes).unwrap(); + + assert_eq!(RntbdResponse::read(&bytes).unwrap(), response); + } + #[test] fn unknown_token_id_is_silently_skipped() { let mut frame = response_header(StatusCode::Ok); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs index e66cd98cbe6..ae166084d46 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs @@ -673,6 +673,26 @@ pub(super) enum RntbdResponseToken { SessionToken, } +impl From for TokenId { + fn from(value: RntbdResponseToken) -> Self { + Self(match value { + RntbdResponseToken::PayloadPresent => 0x0000, + RntbdResponseToken::ContinuationToken => 0x0003, + RntbdResponseToken::ETag => 0x0004, + RntbdResponseToken::RetryAfterMilliseconds => 0x000C, + RntbdResponseToken::Lsn => 0x0013, + RntbdResponseToken::RequestCharge => 0x0015, + RntbdResponseToken::OwnerFullName => 0x0017, + RntbdResponseToken::SubStatus => 0x001C, + RntbdResponseToken::PartitionKeyRangeId => 0x0021, + RntbdResponseToken::ItemLsn => 0x0032, + RntbdResponseToken::GlobalCommittedLsn => 0x0029, + RntbdResponseToken::TransportRequestId => 0x0035, + RntbdResponseToken::SessionToken => 0x003E, + }) + } +} + impl TryFrom for RntbdResponseToken { type Error = (); @@ -820,7 +840,7 @@ impl TryFrom for RntbdOperationType { fn try_from(value: u16) -> azure_core::Result { match value { 0x0001 | 0x0003 | 0x0004 | 0x0005 | 0x0006 | 0x0008 | 0x0009 | 0x000F | 0x0011 - | 0x0012 | 0x0013 | 0x0025 => Ok(Self(value)), + | 0x0012 | 0x0013 | 0x0025 | 0x0042 => Ok(Self(value)), other => Err(data_conversion_error(format!( "unknown RNTBD operation type 0x{other:04X}" ))), @@ -840,6 +860,7 @@ impl TryFrom for OperationType { 0x0006 => Ok(Self::Replace), 0x0008 => Ok(Self::Execute), 0x0009 => Ok(Self::SqlQuery), + 0x0042 => Ok(Self::QueryPlan), 0x000F => Ok(Self::Query), 0x0011 => Ok(Self::Head), 0x0012 => Ok(Self::HeadFeed), diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs index f227ba386f2..8bfa9d68a40 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs @@ -25,6 +25,7 @@ use super::ru_model::RuChargingModel; /// which is what the driver's background refresh loop polls. #[derive(Clone, Debug)] pub struct VirtualAccountConfig { + account_id: String, regions: Vec, write_mode: WriteMode, consistency: ConsistencyLevel, @@ -62,6 +63,7 @@ impl VirtualAccountConfig { } } Ok(Self { + account_id: "emulator-account".to_owned(), regions, write_mode: WriteMode::Single, consistency: ConsistencyLevel::Session, @@ -73,6 +75,19 @@ impl VirtualAccountConfig { }) } + /// Sets the account ID emitted by the hosted account-discovery response. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn with_account_id(mut self, account_id: impl Into) -> Self { + self.account_id = account_id.into(); + self + } + + /// Returns the account ID emitted by account discovery. + pub(crate) fn account_id(&self) -> &str { + &self.account_id + } + /// Sets the write mode. pub fn with_write_mode(mut self, mode: WriteMode) -> Self { self.write_mode = mode; @@ -241,19 +256,20 @@ impl VirtualAccountConfig { let scheme = url.scheme(); let port = url.port_or_known_default(); for r in &self.regions { - let Some(rhost) = r.gateway_url.host_str() else { - continue; + let matches = |candidate: &Url| { + candidate + .host_str() + .is_some_and(|candidate_host| candidate_host.eq_ignore_ascii_case(host)) + && candidate.scheme() == scheme + && candidate.port_or_known_default() == port }; - if !rhost.eq_ignore_ascii_case(host) { - continue; - } - if r.gateway_url.scheme() != scheme { - continue; + if matches(&r.gateway_url) { + return Some(&r.name); } - if r.gateway_url.port_or_known_default() != port { - continue; + #[cfg(feature = "__internal_in_memory_emulator")] + if r.thin_client_url.as_ref().is_some_and(matches) { + return Some(&r.name); } - return Some(&r.name); } None } @@ -273,6 +289,8 @@ impl VirtualAccountConfig { pub struct VirtualRegion { name: String, gateway_url: Url, + #[cfg(feature = "__internal_in_memory_emulator")] + thin_client_url: Option, region_id: u64, } @@ -285,10 +303,20 @@ impl VirtualRegion { Self { name: name.to_string(), gateway_url, + #[cfg(feature = "__internal_in_memory_emulator")] + thin_client_url: None, region_id: 0, } } + /// Configures the Gateway V2 thin-client endpoint for this region. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn with_thin_client_url(mut self, url: Url) -> Self { + self.thin_client_url = Some(url); + self + } + /// Creates a new region with an explicit region ID. pub fn with_region_id(mut self, id: u64) -> Self { self.region_id = id; @@ -303,6 +331,13 @@ impl VirtualRegion { &self.gateway_url } + /// Returns the Gateway V2 thin-client endpoint when hosted externally. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub(crate) fn thin_client_url(&self) -> Option<&Url> { + self.thin_client_url.as_ref() + } + pub fn region_id(&self) -> u64 { self.region_id } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs new file mode 100644 index 00000000000..76b1c0500bc --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs @@ -0,0 +1,557 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Hosted Gateway V2 adapter for the in-memory emulator. + +use azure_core::{ + http::{ + headers::{HeaderName, HeaderValue, Headers}, + AsyncRawResponse, Method, Request, StatusCode, + }, + Bytes, +}; +use uuid::Uuid; + +use crate::{ + driver::transport::rntbd::{ + tokens::{RntbdRequestToken, TokenValue}, + RntbdRequestFrame, RntbdResponse, + }, + models::{CosmosStatus, OperationType, ResourceType}, +}; + +use super::{ConsistencyLevel, InMemoryEmulatorHttpClient}; + +impl InMemoryEmulatorHttpClient { + /// Executes a hosted Gateway V2 request and returns an RNTBD-framed response. + #[doc(hidden)] + pub async fn execute_gateway_v2_request( + &self, + request: &Request, + ) -> crate::error::Result { + let request_body: Bytes = request.body().into(); + let frame = + RntbdRequestFrame::read(request_body.as_ref()).map_err(gateway_v2_bad_request)?; + let activity_id = frame.activity_id; + let request = decode_request(request, frame, self.store().config().consistency())?; + let response = self.execute_request(&request).await?; + encode_response(response, activity_id).await + } +} + +#[derive(Default)] +struct RequestMetadata { + database: Option, + collection: Option, + document: Option, + partition_key: Option, + partition_key_range_id: Option, + continuation: Option, + session_token: Option, + match_condition: Option, + effective_partition_key: Option, + start_epk: Option, + end_epk: Option, + page_size: Option, + return_minimal: bool, + supported_query_features: Option, + query_version: Option, + allow_tentative_writes: bool, + consistency_level: Option, + read_consistency_strategy: Option, +} + +fn decode_request( + outer_request: &Request, + frame: RntbdRequestFrame, + account_consistency: ConsistencyLevel, +) -> crate::error::Result { + if frame.resource_type != ResourceType::Document { + return Err(gateway_v2_bad_request(format!( + "hosted Gateway V2 supports Document resources, got {:?}", + frame.resource_type + ))); + } + let mut metadata = decode_metadata(frame.metadata); + if metadata.allow_tentative_writes { + return Err(gateway_v2_bad_request( + "hosted Gateway V2 does not yet support AllowTentativeWrites", + )); + } + if metadata.read_consistency_strategy.is_some() { + return Err(gateway_v2_bad_request( + "hosted Gateway V2 does not yet support non-default ReadConsistencyStrategy", + )); + } + if let Some(value) = metadata.consistency_level { + if !matches!(value, 0x00..=0x04) { + return Err(gateway_v2_bad_request( + "RNTBD request contains an unknown ConsistencyLevel value", + )); + } + if value != consistency_wire_byte(account_consistency) { + return Err(gateway_v2_bad_request( + "hosted Gateway V2 does not yet support per-request consistency overrides", + )); + } + } + if matches!( + frame.operation_type, + OperationType::Query | OperationType::SqlQuery | OperationType::ReadFeed + ) && metadata.start_epk.is_none() + { + if let Some(effective_partition_key) = metadata.effective_partition_key.as_ref() { + metadata.start_epk = Some(effective_partition_key.clone()); + metadata.end_epk = Some(format!("{effective_partition_key}FF")); + } + } + if matches!( + frame.operation_type, + OperationType::Read | OperationType::Replace | OperationType::Delete + ) && metadata.partition_key.is_none() + && metadata.effective_partition_key.is_some() + { + return Err(gateway_v2_bad_request( + "hosted Gateway V2 requires the string PartitionKey token for point operations", + )); + } + tracing::debug!( + operation = ?frame.operation_type, + partition_key_range_id = ?metadata.partition_key_range_id, + start_epk = ?metadata.start_epk, + end_epk = ?metadata.end_epk, + "decoded hosted Gateway V2 request target" + ); + let database = metadata + .database + .as_deref() + .ok_or_else(|| gateway_v2_bad_request("RNTBD request is missing DatabaseName"))?; + let collection = metadata + .collection + .as_deref() + .ok_or_else(|| gateway_v2_bad_request("RNTBD request is missing CollectionName"))?; + + let (method, document_required) = match frame.operation_type { + OperationType::Create | OperationType::Upsert => (Method::Post, false), + OperationType::Read => (Method::Get, true), + OperationType::Replace => (Method::Put, true), + OperationType::Delete => (Method::Delete, true), + OperationType::Query + | OperationType::SqlQuery + | OperationType::QueryPlan + | OperationType::Batch => (Method::Post, false), + OperationType::ReadFeed => (Method::Get, false), + operation => { + return Err(gateway_v2_bad_request(format!( + "unsupported hosted Gateway V2 operation {operation:?}" + ))) + } + }; + + let mut url = outer_request.url().clone(); + url.set_query(None); + url.set_fragment(None); + { + let mut segments = url + .path_segments_mut() + .map_err(|_| gateway_v2_bad_request("thin-client endpoint cannot be a base URL"))?; + segments + .clear() + .push("dbs") + .push(database) + .push("colls") + .push(collection) + .push("docs"); + if document_required { + let document = metadata.document.as_deref().ok_or_else(|| { + gateway_v2_bad_request("RNTBD point request is missing DocumentName") + })?; + segments.push(document); + } + } + + let mut request = Request::new(url, method); + request.headers_mut().insert( + "x-ms-activity-id", + HeaderValue::from(frame.activity_id.to_string()), + ); + if let Some(value) = metadata.partition_key { + request + .headers_mut() + .insert("x-ms-documentdb-partitionkey", value); + } + if let Some(value) = metadata.partition_key_range_id { + request + .headers_mut() + .insert("x-ms-documentdb-partitionkeyrangeid", value); + } + if let Some(value) = metadata.continuation { + request.headers_mut().insert("x-ms-continuation", value); + } + if let Some(value) = metadata.session_token { + request.headers_mut().insert("x-ms-session-token", value); + } + if let Some(value) = metadata.page_size { + let value = if value == u32::MAX { + "-1".to_owned() + } else { + value.to_string() + }; + request.headers_mut().insert("x-ms-max-item-count", value); + } + if let Some(value) = metadata.match_condition { + let header = match frame.operation_type { + OperationType::Read | OperationType::ReadFeed => "if-none-match", + _ => "if-match", + }; + request.headers_mut().insert(header, value); + } + if let Some(value) = metadata.start_epk { + request.headers_mut().insert("x-ms-start-epk", value); + request + .headers_mut() + .insert("x-ms-read-key-type", "EffectivePartitionKeyRange"); + } + if let Some(value) = metadata.end_epk { + request.headers_mut().insert("x-ms-end-epk", value); + request + .headers_mut() + .insert("x-ms-read-key-type", "EffectivePartitionKeyRange"); + } + if metadata.return_minimal { + request.headers_mut().insert("prefer", "return=minimal"); + } + if let Some(value) = metadata.supported_query_features { + request + .headers_mut() + .insert("x-ms-cosmos-supported-query-features", value); + } + if let Some(value) = metadata.query_version { + request + .headers_mut() + .insert("x-ms-cosmos-query-version", value); + } + + match frame.operation_type { + OperationType::Upsert => request + .headers_mut() + .insert("x-ms-documentdb-is-upsert", "true"), + OperationType::Query | OperationType::SqlQuery => request + .headers_mut() + .insert("x-ms-documentdb-isquery", "true"), + OperationType::QueryPlan => request + .headers_mut() + .insert("x-ms-cosmos-is-query-plan-request", "true"), + OperationType::Batch => request + .headers_mut() + .insert("x-ms-cosmos-is-batch-request", "true"), + _ => {} + } + if let Some(body) = frame.body { + request.set_body(body); + } + Ok(request) +} + +fn decode_metadata(tokens: Vec) -> RequestMetadata { + let mut metadata = RequestMetadata::default(); + for token in tokens { + let Ok(kind) = RntbdRequestToken::try_from(token.id.value()) else { + continue; + }; + match kind { + RntbdRequestToken::DatabaseName => metadata.database = token_string(token.value), + RntbdRequestToken::CollectionName => metadata.collection = token_string(token.value), + RntbdRequestToken::DocumentName => metadata.document = token_string(token.value), + RntbdRequestToken::PartitionKey => metadata.partition_key = token_string(token.value), + RntbdRequestToken::PartitionKeyRangeId => { + metadata.partition_key_range_id = token_string(token.value) + } + RntbdRequestToken::ContinuationToken => { + metadata.continuation = token_string(token.value) + } + RntbdRequestToken::SessionToken => metadata.session_token = token_string(token.value), + RntbdRequestToken::Match => metadata.match_condition = token_string(token.value), + RntbdRequestToken::StartEpkHash => metadata.start_epk = token_hex(token.value), + RntbdRequestToken::EndEpkHash => metadata.end_epk = token_hex(token.value), + RntbdRequestToken::EffectivePartitionKey => { + metadata.effective_partition_key = token_hex(token.value) + } + RntbdRequestToken::PageSize => { + if let TokenValue::ULong(value) = token.value { + metadata.page_size = Some(value); + } + } + RntbdRequestToken::ReturnPreference => { + metadata.return_minimal = + matches!(token.value, TokenValue::Byte(value) if value != 0) + } + RntbdRequestToken::SupportedQueryFeatures => { + metadata.supported_query_features = token_string(token.value) + } + RntbdRequestToken::QueryVersion => metadata.query_version = token_string(token.value), + RntbdRequestToken::AllowTentativeWrites => { + metadata.allow_tentative_writes = + matches!(token.value, TokenValue::Byte(value) if value != 0) + } + RntbdRequestToken::ConsistencyLevel => { + if let TokenValue::Byte(value) = token.value { + metadata.consistency_level = Some(value); + } + } + RntbdRequestToken::ReadConsistencyStrategy => { + if let TokenValue::Byte(value) = token.value { + metadata.read_consistency_strategy = Some(value); + } + } + RntbdRequestToken::ResourceId + | RntbdRequestToken::AuthorizationToken + | RntbdRequestToken::PayloadPresent + | RntbdRequestToken::Date + | RntbdRequestToken::CollectionRid + | RntbdRequestToken::TransportRequestId + | RntbdRequestToken::SDKSupportedCapabilities + | RntbdRequestToken::GlobalDatabaseAccountName => { + // The hosted adapter already has the corresponding routing or + // transport context; these tokens do not alter store semantics. + } + } + } + metadata +} + +fn token_string(value: TokenValue) -> Option { + match value { + TokenValue::SmallString(value) + | TokenValue::String(value) + | TokenValue::ULongString(value) => Some(value), + _ => None, + } +} + +fn token_hex(value: TokenValue) -> Option { + let bytes = match value { + TokenValue::SmallBytes(value) + | TokenValue::Bytes(value) + | TokenValue::ULongBytes(value) => value, + _ => return None, + }; + Some(bytes.iter().map(|byte| format!("{byte:02X}")).collect()) +} + +fn consistency_wire_byte(value: ConsistencyLevel) -> u8 { + match value { + ConsistencyLevel::Strong => 0x00, + ConsistencyLevel::BoundedStaleness => 0x01, + ConsistencyLevel::Session => 0x02, + ConsistencyLevel::Eventual => 0x03, + ConsistencyLevel::ConsistentPrefix => 0x04, + } +} + +async fn encode_response( + response: AsyncRawResponse, + request_activity_id: Uuid, +) -> crate::error::Result { + let response = response + .try_into_raw_response() + .await + .map_err(gateway_v2_internal_error)?; + let headers = response.headers(); + let status = header_u32(headers, "x-ms-substatus") + .map(|sub_status| CosmosStatus::new(response.status()).with_sub_status(sub_status as u16)) + .unwrap_or_else(|| CosmosStatus::new(response.status())); + let activity_id = header_string(headers, "x-ms-activity-id") + .and_then(|value| Uuid::parse_str(&value).ok()) + .unwrap_or(request_activity_id); + let rntbd = RntbdResponse { + status, + activity_id, + body: response.body().as_ref().to_vec(), + continuation_token: header_string(headers, "x-ms-continuation"), + etag: header_string(headers, "etag"), + retry_after_ms: header_u32(headers, "x-ms-retry-after-ms"), + lsn: header_i64(headers, "lsn"), + request_charge: header_f64(headers, "x-ms-request-charge"), + owner_full_name: header_string(headers, "x-ms-alt-content-path"), + partition_key_range_id: header_string(headers, "x-ms-documentdb-partitionkeyrangeid"), + item_lsn: header_i64(headers, "x-ms-item-lsn"), + global_committed_lsn: header_i64(headers, "x-ms-global-committed-lsn"), + transport_request_id: header_u32(headers, "x-ms-transport-request-id"), + session_token: header_string(headers, "x-ms-session-token"), + }; + let mut body = Vec::new(); + rntbd.write(&mut body).map_err(gateway_v2_internal_error)?; + let mut outer_headers = Headers::new(); + outer_headers.insert("content-type", "application/octet-stream"); + for name in [ + "x-ms-request-duration-ms", + "lsn", + "x-ms-item-lsn", + "x-ms-global-committed-lsn", + "x-ms-documentdb-query-metrics", + "x-ms-index-utilization", + ] { + if let Some(value) = header_string(headers, name) { + outer_headers.insert(name, value); + } + } + Ok(AsyncRawResponse::from_bytes( + StatusCode::Ok, + outer_headers, + body, + )) +} + +fn header_string(headers: &Headers, name: &'static str) -> Option { + headers + .get_optional_str(&HeaderName::from_static(name)) + .map(str::to_owned) +} + +fn header_u32(headers: &Headers, name: &'static str) -> Option { + header_string(headers, name)?.parse().ok() +} + +fn header_i64(headers: &Headers, name: &'static str) -> Option { + header_string(headers, name)?.parse().ok() +} + +fn header_f64(headers: &Headers, name: &'static str) -> Option { + header_string(headers, name)?.parse().ok() +} + +fn gateway_v2_bad_request(error: impl std::fmt::Display) -> crate::error::CosmosError { + crate::error::CosmosError::builder() + .with_status(CosmosStatus::new(StatusCode::BadRequest)) + .with_message(error.to_string()) + .build() +} + +fn gateway_v2_internal_error(error: impl std::fmt::Display) -> crate::error::CosmosError { + crate::error::CosmosError::builder() + .with_status(CosmosStatus::new(StatusCode::InternalServerError)) + .with_message(error.to_string()) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + driver::transport::rntbd::Token, + in_memory_emulator::{ContainerConfig, VirtualAccountConfig, VirtualRegion}, + models::PartitionKeyDefinition, + }; + use url::Url; + + #[tokio::test] + async fn create_item_round_trips_through_gateway_v2() { + let thin_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", "http://127.0.0.1:18081/".parse().unwrap()) + .with_thin_client_url(thin_url.clone()); + let emulator = + InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![region]).unwrap()); + let store = emulator.store(); + store.create_database("db"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + store.create_container_with_config( + "db", + "coll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + + let activity_id = Uuid::new_v4(); + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Create, + activity_id, + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::partition_key(r#"["pk1"]"#.to_owned()), + Token::payload_present(true), + ], + body: Some( + serde_json::to_vec(&serde_json::json!({ + "id": "item1", "pk": "pk1", "value": 42 + })) + .unwrap(), + ), + }; + let mut bytes = Vec::new(); + frame.write(&mut bytes).unwrap(); + let mut request = Request::new(thin_url, Method::Post); + request.set_body(bytes); + + let response = emulator + .execute_gateway_v2_request(&request) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::Ok); + let response = RntbdResponse::read(response.body().as_ref()).unwrap(); + assert_eq!(response.status.status_code(), StatusCode::Created); + assert_eq!(response.activity_id, activity_id); + assert!(!response.body.is_empty()); + } + + #[test] + fn effective_partition_key_scopes_query_range() { + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Query, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::effective_partition_key(vec![0x10, 0x20]), + ], + body: Some(br#"{"query":"SELECT * FROM c"}"#.to_vec()), + }; + let outer = Request::new(Url::parse("http://127.0.0.1:18444/").unwrap(), Method::Post); + + let request = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap(); + assert_eq!( + request + .headers() + .get_optional_str(&HeaderName::from_static("x-ms-start-epk")), + Some("1020") + ); + assert_eq!( + request + .headers() + .get_optional_str(&HeaderName::from_static("x-ms-end-epk")), + Some("1020FF") + ); + } + + #[test] + fn tentative_writes_are_rejected_explicitly() { + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Create, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::allow_tentative_writes(true), + ], + body: Some(br#"{"id":"item"}"#.to_vec()), + }; + let outer = Request::new(Url::parse("http://127.0.0.1:18444/").unwrap(), Method::Post); + + let error = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap_err(); + assert!(error.to_string().contains("AllowTentativeWrites")); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/mod.rs index c2dc807e0dd..18ded73141e 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/mod.rs @@ -31,6 +31,8 @@ mod client; mod config; mod dispatch; mod epk; +#[cfg(feature = "__internal_in_memory_emulator")] +mod gateway_v2; mod observer; mod operations; mod response; @@ -50,4 +52,4 @@ pub use observer::RequestObserver; #[doc(hidden)] pub use response::headers as test_headers; pub use ru_model::RuChargingModel; -pub use store::EmulatorStore; +pub use store::{EmulatorStore, ManualControlPlaneOperation}; diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs index 63987b1f76a..eaf3d65c264 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs @@ -140,7 +140,7 @@ pub(crate) async fn handle_operation( store: &Arc, region_name: &str, parsed: &ParsedRequest, - request_headers: &Headers, + _request_headers: &Headers, request_body: &[u8], ) -> AsyncRawResponse { let start = Instant::now(); @@ -273,8 +273,14 @@ pub(crate) async fn handle_operation( } #[cfg(feature = "preview_dtx")] OperationType::DistributedTransaction => { - handle_distributed_transaction(store, region_name, request_headers, request_body, start) - .await + handle_distributed_transaction( + store, + region_name, + _request_headers, + request_body, + start, + ) + .await } OperationType::BadRequestPath(desc) => bad_request_path_response(desc, start), OperationType::InvalidInput(desc) => invalid_input_response(desc, start), @@ -4559,6 +4565,7 @@ fn handle_read( .ru_model() .compute_read_ru(doc.body_size_bytes); let lsn = partition.current_lsn(); + let item_lsn = doc.lsn; let body = doc.body.clone(); let etag = doc.etag.clone(); drop(docs); @@ -4575,11 +4582,16 @@ fn handle_read( .with_etag(&etag); return Err(decorate_point_response(builder, headers, Some(lsn)).build()); } - return Ok((body, etag, token, charge, lsn, headers)); + return Ok((body, etag, token, charge, lsn, item_lsn, headers)); } } - Err(error_response( + let lsn = partition.current_lsn(); + let headers = Some(PointResponseHeaders::from_partition( + partition, + store.next_transport_request_id(), + )); + let builder = error_response( StatusCode::NotFound, None, "NotFound", @@ -4591,15 +4603,16 @@ fn handle_read( &token, start, ) - .build()) + .with_lsn(lsn); + Err(decorate_point_response(builder, headers, Some(lsn)).build()) }); match result { - Some(Ok((body, etag, token, charge, lsn, headers))) => { + Some(Ok((body, etag, token, charge, lsn, item_lsn, headers))) => { let builder = success_response(StatusCode::Ok, &body, charge, &token, start) .with_etag(&etag) .with_lsn(lsn); - decorate_point_response(builder, headers, Some(lsn)).build() + decorate_point_response(builder, headers, Some(item_lsn)).build() } Some(Err(response)) => response, None => container_not_found(db_id, coll_id, start), @@ -5053,6 +5066,23 @@ async fn handle_upsert_locked( let (new_doc, status, charge) = { let mut docs = partition.documents.write().unwrap(); let logical = docs.entry(epk.clone()).or_default(); + if let Some(if_match) = parsed.if_match.as_ref() { + if logical + .get(&doc_id) + .is_none_or(|existing| *if_match != existing.etag) + { + return Err(error_response( + StatusCode::PreconditionFailed, + None, + "PreconditionFailed", + "One of the specified pre-condition is not met.", + 1.0, + "", + start, + ) + .build()); + } + } let (status, rid, self_link) = match logical.get(&doc_id) { Some(existing) => ( StatusCode::Ok, diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs index 92055ca909f..04600070344 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs @@ -203,6 +203,7 @@ impl ResponseBuilder { self } + #[cfg(feature = "preview_dtx")] pub fn without_header(mut self, name: HeaderName) -> Self { self.headers.remove(name); self diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs index 87ed7ee1714..fe4d02b05e3 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs @@ -40,6 +40,54 @@ impl ControlPlaneTaskKey { } } +enum ControlPlaneProgression { + Automatic(Duration), + Manual(futures::channel::oneshot::Receiver<()>), +} + +impl ControlPlaneProgression { + async fn wait(self) -> bool { + match self { + Self::Automatic(duration) => { + if !duration.is_zero() { + tokio::time::sleep(duration).await; + } + true + } + Self::Manual(release) => release.await.is_ok(), + } + } +} + +/// Handle for a manually progressed split or merge operation. +#[doc(hidden)] +pub struct ManualControlPlaneOperation { + release: Option>, + completed: futures::channel::oneshot::Receiver, +} + +impl ManualControlPlaneOperation { + /// Releases the operation's partition lock and waits for topology replacement. + pub async fn complete(mut self) -> crate::error::Result<()> { + self.release + .take() + .ok_or_else(|| host_control_plane_error("manual operation was already released"))? + .send(()) + .map_err(|_| host_control_plane_error("manual operation task ended before release"))?; + if self + .completed + .await + .map_err(|_| host_control_plane_error("manual operation ended without a result"))? + { + Ok(()) + } else { + Err(host_control_plane_error( + "manual operation could not update the requested partitions", + )) + } + } +} + /// Applies a single document mutation to a partition under LWW /// (Last-Writer-Wins) on `(_ts, lsn)`. /// @@ -1783,6 +1831,208 @@ impl ThroughputTracker { // --- Split / Merge --- impl EmulatorStore { + /// Returns the EPK boundary a midpoint split would use for a physical partition. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn midpoint_split_epk( + &self, + db_id: &str, + coll_id: &str, + partition_id: u32, + ) -> crate::error::Result { + let region = self + .region(self.config.write_region_name()) + .ok_or_else(|| host_control_plane_error("write region does not exist"))?; + region + .with_container(db_id, coll_id, |state| { + let partition = state + .physical_partitions + .iter() + .find(|partition| partition.id == partition_id) + .ok_or_else(|| { + host_control_plane_error(format!( + "partition {partition_id} does not exist in {db_id}/{coll_id}" + )) + })?; + compute_epk_midpoint( + &partition.epk_min, + &partition.epk_max, + state.metadata.partition_key.kind(), + state.metadata.partition_key.version(), + ) + .map_err(host_control_plane_error) + }) + .ok_or_else(|| { + host_control_plane_error(format!("container {db_id}/{coll_id} does not exist")) + })? + } + + /// Returns a split EPK that most closely balances serialized document bytes. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn storage_split_epk( + &self, + db_id: &str, + coll_id: &str, + partition_id: u32, + ) -> crate::error::Result { + let region = self + .region(self.config.write_region_name()) + .ok_or_else(|| host_control_plane_error("write region does not exist"))?; + region + .with_container(db_id, coll_id, |state| { + let partition = state + .physical_partitions + .iter() + .find(|partition| partition.id == partition_id) + .ok_or_else(|| { + host_control_plane_error(format!( + "partition {partition_id} does not exist in {db_id}/{coll_id}" + )) + })?; + let documents = partition.documents.read().unwrap(); + let groups: Vec<(Epk, u64)> = documents + .iter() + .map(|(epk, documents)| { + let bytes = documents + .values() + .map(|document| document.body_size_bytes.max(1) as u64) + .sum(); + (epk.clone(), bytes) + }) + .collect(); + if groups.len() < 2 { + return Err(host_control_plane_error( + "storage split requires documents in at least two distinct EPK groups", + )); + } + + let total_bytes: u64 = groups.iter().map(|(_, bytes)| bytes).sum(); + let mut left_bytes = groups[0].1; + let mut best = (total_bytes.abs_diff(left_bytes.saturating_mul(2)), 1usize); + for (index, (_, bytes)) in groups.iter().enumerate().skip(1).take(groups.len() - 2) + { + left_bytes = left_bytes.saturating_add(*bytes); + let score = total_bytes.abs_diff(left_bytes.saturating_mul(2)); + if score < best.0 { + best = (score, index + 1); + } + } + Ok(groups[best.1].0.clone()) + }) + .ok_or_else(|| { + host_control_plane_error(format!("container {db_id}/{coll_id} does not exist")) + })? + } + + /// Starts a storage-balanced split and returns the selected EPK boundary. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn split_partition_by_storage( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id: u32, + min_lock_duration: Duration, + ) -> crate::error::Result { + let split_epk = self.storage_split_epk(db_id, coll_id, partition_id)?; + self.split_partition_at_epk( + db_id, + coll_id, + partition_id, + split_epk.clone(), + min_lock_duration, + ); + Ok(split_epk) + } + + /// Returns the IDs of partitions whose parent list contains all supplied IDs. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn child_partition_ids(&self, db_id: &str, coll_id: &str, parent_ids: &[u32]) -> Vec { + let Some(region) = self.region(self.config.write_region_name()) else { + return Vec::new(); + }; + let mut children = region + .with_container(db_id, coll_id, |state| { + state + .physical_partitions + .iter() + .filter(|partition| { + parent_ids + .iter() + .all(|parent_id| partition.parents.contains(parent_id)) + }) + .map(|partition| partition.id) + .collect::>() + }) + .unwrap_or_default(); + children.sort_unstable(); + children + } + + /// Splits at an explicit EPK and awaits this specific control-plane operation. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub async fn split_partition_and_wait( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id: u32, + split_epk: Epk, + min_lock_duration: Duration, + ) -> crate::error::Result> { + let (completion, completed) = futures::channel::oneshot::channel(); + self.split_partition_internal( + db_id, + coll_id, + partition_id, + Some(split_epk), + ControlPlaneProgression::Automatic(min_lock_duration), + Some(completion), + ); + if !completed + .await + .map_err(|_| host_control_plane_error("split task ended before reporting a result"))? + { + return Err(host_control_plane_error( + "split did not complete; verify the partition and EPK boundary", + )); + } + let children = self.child_partition_ids(db_id, coll_id, &[partition_id]); + if children.len() != 2 { + return Err(host_control_plane_error( + "split completed without producing exactly two child partitions", + )); + } + Ok(children) + } + + /// Locks a partition for a split that completes only when its handle is released. + #[doc(hidden)] + pub fn begin_manual_split_partition( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id: u32, + split_epk: Epk, + ) -> ManualControlPlaneOperation { + let (release, released) = futures::channel::oneshot::channel(); + let (completion, completed) = futures::channel::oneshot::channel(); + self.split_partition_internal( + db_id, + coll_id, + partition_id, + Some(split_epk), + ControlPlaneProgression::Manual(released), + Some(completion), + ); + ManualControlPlaneOperation { + release: Some(release), + completed, + } + } + /// Splits a physical partition into two child partitions. /// /// During `min_lock_duration` (plus doc redistribution time), operations on the @@ -1797,7 +2047,14 @@ impl EmulatorStore { partition_id: u32, min_lock_duration: Duration, ) { - self.split_partition_internal(db_id, coll_id, partition_id, None, min_lock_duration); + self.split_partition_internal( + db_id, + coll_id, + partition_id, + None, + ControlPlaneProgression::Automatic(min_lock_duration), + None, + ); } /// Splits a physical partition at an explicit EPK boundary. Test-only. @@ -1815,7 +2072,8 @@ impl EmulatorStore { coll_id, partition_id, Some(split_epk), - min_lock_duration, + ControlPlaneProgression::Automatic(min_lock_duration), + None, ); } @@ -1825,7 +2083,8 @@ impl EmulatorStore { coll_id: &str, partition_id: u32, split_epk: Option, - min_lock_duration: Duration, + progression: ControlPlaneProgression, + completion: Option>, ) { // Lock the partition in all regions { @@ -1855,20 +2114,36 @@ impl EmulatorStore { coll: coll_id.to_string(), partitions: vec![partition_id], }; + let track_in_test_registry = completion.is_none(); let handle = tokio::spawn(async move { let _guard = lock.lock().await; - if !min_lock_duration.is_zero() { - tokio::time::sleep(min_lock_duration).await; + if !progression.wait().await { + store.unlock_partitions(&(db.clone(), coll.clone()), &[partition_id]); + if let Some(completion) = completion { + let _ = completion.send(false); + } + return; } // execute_split does the actual doc redistribution under the lock, // then unlocks partitions when done - store.execute_split(&db, &coll, partition_id, split_epk); + let succeeded = store.execute_split(&db, &coll, partition_id, split_epk); + if let Some(completion) = completion { + let _ = completion.send(succeeded); + } }); - self.control_plane_tasks.lock().unwrap().push((key, handle)); + if track_in_test_registry { + self.control_plane_tasks.lock().unwrap().push((key, handle)); + } } /// Performs the actual split after the lock period. - fn execute_split(&self, db_id: &str, coll_id: &str, partition_id: u32, split_epk: Option) { + fn execute_split( + &self, + db_id: &str, + coll_id: &str, + partition_id: u32, + split_epk: Option, + ) -> bool { // Local-only enum used to ferry preview state out of a regions read // guard so we can drop the guard before re-acquiring it on the abort // path. Avoids recursive same-thread RwLock::read (unspecified in std). @@ -2007,9 +2282,9 @@ impl EmulatorStore { } } } - return; + return false; } - None => return, + None => return false, }) else { unreachable!() @@ -2140,6 +2415,7 @@ impl EmulatorStore { // child IDs were allocated. } } + true } /// Merges two adjacent physical partitions into one child partition. @@ -2156,6 +2432,87 @@ impl EmulatorStore { partition_id_a: u32, partition_id_b: u32, min_lock_duration: Duration, + ) { + self.merge_partitions_internal( + db_id, + coll_id, + partition_id_a, + partition_id_b, + ControlPlaneProgression::Automatic(min_lock_duration), + None, + ); + } + + /// Merges two adjacent partitions and awaits this specific operation. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub async fn merge_partitions_and_wait( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id_a: u32, + partition_id_b: u32, + min_lock_duration: Duration, + ) -> crate::error::Result { + let (completion, completed) = futures::channel::oneshot::channel(); + self.merge_partitions_internal( + db_id, + coll_id, + partition_id_a, + partition_id_b, + ControlPlaneProgression::Automatic(min_lock_duration), + Some(completion), + ); + if !completed + .await + .map_err(|_| host_control_plane_error("merge task ended before reporting a result"))? + { + return Err(host_control_plane_error( + "merge did not complete; verify that both partitions exist and are adjacent", + )); + } + let children = self.child_partition_ids(db_id, coll_id, &[partition_id_a, partition_id_b]); + match children.as_slice() { + [child] => Ok(*child), + _ => Err(host_control_plane_error( + "merge completed without producing exactly one child partition", + )), + } + } + + /// Locks two partitions for a merge that completes only when its handle is released. + #[doc(hidden)] + pub fn begin_manual_merge_partitions( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id_a: u32, + partition_id_b: u32, + ) -> ManualControlPlaneOperation { + let (release, released) = futures::channel::oneshot::channel(); + let (completion, completed) = futures::channel::oneshot::channel(); + self.merge_partitions_internal( + db_id, + coll_id, + partition_id_a, + partition_id_b, + ControlPlaneProgression::Manual(released), + Some(completion), + ); + ManualControlPlaneOperation { + release: Some(release), + completed, + } + } + + fn merge_partitions_internal( + self: &Arc, + db_id: &str, + coll_id: &str, + partition_id_a: u32, + partition_id_b: u32, + progression: ControlPlaneProgression, + completion: Option>, ) { // Lock both partitions in all regions { @@ -2183,14 +2540,27 @@ impl EmulatorStore { coll: coll_id.to_string(), partitions: vec![partition_id_a, partition_id_b], }; + let track_in_test_registry = completion.is_none(); let handle = tokio::spawn(async move { let _guard = lock.lock().await; - if !min_lock_duration.is_zero() { - tokio::time::sleep(min_lock_duration).await; + if !progression.wait().await { + store.unlock_partitions( + &(db.clone(), coll.clone()), + &[partition_id_a, partition_id_b], + ); + if let Some(completion) = completion { + let _ = completion.send(false); + } + return; + } + let succeeded = store.execute_merge(&db, &coll, partition_id_a, partition_id_b); + if let Some(completion) = completion { + let _ = completion.send(succeeded); } - store.execute_merge(&db, &coll, partition_id_a, partition_id_b); }); - self.control_plane_tasks.lock().unwrap().push((key, handle)); + if track_in_test_registry { + self.control_plane_tasks.lock().unwrap().push((key, handle)); + } } fn unlock_partitions(&self, key: &(String, String), partition_ids: &[u32]) { @@ -2208,7 +2578,13 @@ impl EmulatorStore { } /// Performs the actual merge after the lock period. - fn execute_merge(&self, db_id: &str, coll_id: &str, partition_id_a: u32, partition_id_b: u32) { + fn execute_merge( + &self, + db_id: &str, + coll_id: &str, + partition_id_a: u32, + partition_id_b: u32, + ) -> bool { enum MergePreview { Ready((Epk, Epk, u64, u32, String, Option)), NonAdjacent(Epk, Epk), @@ -2281,11 +2657,11 @@ impl EmulatorStore { "in-memory emulator: rejecting merge for non-adjacent partitions", ); self.unlock_partitions(&key, &[partition_id_a, partition_id_b]); - return; + return false; } None => { self.unlock_partitions(&key, &[partition_id_a, partition_id_b]); - return; + return false; } }; @@ -2451,9 +2827,20 @@ impl EmulatorStore { state.physical_partitions.push(child); } } + true } } +#[cfg(feature = "__internal_in_memory_emulator")] +fn host_control_plane_error(message: impl Into) -> crate::error::CosmosError { + crate::error::CosmosError::builder() + .with_status(crate::error::CosmosStatus::new( + azure_core::http::StatusCode::BadRequest, + )) + .with_message(message.into()) + .build() +} + /// Computes the EPK midpoint between two EPK bounds (hex strings). /// /// Returns `Err` if either bound is not parseable. Only V2 `Hash` / @@ -2649,6 +3036,55 @@ fn decode_v1_number_hex_to_u32(hex: &str) -> Result { mod tests { use super::*; + #[tokio::test] + async fn manual_split_waits_for_explicit_completion() { + use crate::models::PartitionKeyDefinition; + + let config = super::super::config::VirtualAccountConfig::new(vec![ + super::super::config::VirtualRegion::new( + "r1", + url::Url::parse("https://r1.local").unwrap(), + ), + ]) + .unwrap(); + let store = EmulatorStore::new(config); + store.create_database("db"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + store.create_container_with_config( + "db", + "c", + partition_key, + super::super::config::ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + + let split_epk = store.midpoint_split_epk("db", "c", 0).unwrap(); + let operation = store.begin_manual_split_partition("db", "c", 0, split_epk); + let locked = store + .region("r1") + .unwrap() + .with_container("db", "c", |state| { + state + .physical_partitions + .iter() + .find(|partition| partition.id == 0) + .unwrap() + .is_locked() + }) + .unwrap(); + assert!(locked); + assert!(store.child_partition_ids("db", "c", &[0]).is_empty()); + + operation.complete().await.unwrap(); + + assert_eq!(store.child_partition_ids("db", "c", &[0]).len(), 2); + } + #[test] fn partitions_distribute_across_full_v2_epk_space() { use super::super::epk::{compute_epk, parse_partition_key_header}; diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs index 1cf9ec63b83..7d858c7f58c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs @@ -189,8 +189,8 @@ pub(crate) fn account_properties_to_json( } }; - serde_json::json!({ - "id": "emulator-account", + let mut response = serde_json::json!({ + "id": config.account_id(), "_rid": "emulator.documents.azure.com", "_self": "", "media": "//media/", @@ -211,5 +211,49 @@ pub(crate) fn account_properties_to_json( "systemReplicationPolicy": { "minReplicaSetSize": 3, "maxReplicasetSize": 4 }, "readPolicy": { "primaryReadCoefficient": 1, "secondaryReadCoefficient": 1 }, "queryEngineConfiguration": "{}" - }) + }); + + #[cfg(feature = "__internal_in_memory_emulator")] + { + let thin_client_readable: Vec<_> = config + .regions() + .iter() + .filter_map(|region| { + region.thin_client_url().map(|url| { + serde_json::json!({ + "name": region.name(), + "databaseAccountEndpoint": url.as_str() + }) + }) + }) + .collect(); + let thin_client_writable: Vec<_> = config + .regions() + .iter() + .filter(|region| config.is_write_region(region.name())) + .filter_map(|region| { + region.thin_client_url().map(|url| { + serde_json::json!({ + "name": region.name(), + "databaseAccountEndpoint": url.as_str() + }) + }) + }) + .collect(); + if !thin_client_readable.is_empty() { + let object = response + .as_object_mut() + .expect("account properties response is a JSON object"); + object.insert( + "thinClientReadableLocations".to_owned(), + serde_json::Value::Array(thin_client_readable), + ); + object.insert( + "thinClientWritableLocations".to_owned(), + serde_json::Value::Array(thin_client_writable), + ); + } + } + + response } diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs index 5e9df18391c..482edc40ffd 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs @@ -20,8 +20,12 @@ use std::time::{Duration, Instant}; /// A read operation should succeed because the fault never fires. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_probability_zero_never_fails() -> Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -82,8 +86,12 @@ pub async fn fault_injection_probability_zero_never_fails() -> Result<(), Box Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -141,8 +149,12 @@ pub async fn fault_injection_service_unavailable_causes_failure() -> Result<(), /// A rule targeting only ReadItem should not affect CreateItem operations. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_operation_type_filter() -> Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -206,8 +218,12 @@ pub async fn fault_injection_operation_type_filter() -> Result<(), Box Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -285,8 +301,12 @@ pub async fn fault_injection_hit_limit_stops_after_n_faults() -> Result<(), Box< /// Tests that a ConnectionError fault causes read failures. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_connection_error() -> Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -1073,8 +1093,12 @@ pub async fn gateway_v2_449_retry_with_succeeds_after_hit_limit() -> Result<(), /// retry path instead of driving the loop directly. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn fault_injection_429_honors_configurable_throttle_retry_count( ) -> Result<(), Box> { @@ -1173,8 +1197,12 @@ pub async fn fault_injection_429_honors_configurable_throttle_retry_count( /// invalidation. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn pkrange_refresh_transient_failure_preserves_cached_routing_map( ) -> Result<(), Box> { diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_hedging_kill_switch.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_hedging_kill_switch.rs index a830afe9d81..cf10d43d579 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_hedging_kill_switch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_hedging_kill_switch.rs @@ -71,8 +71,12 @@ struct TestItem { /// → runtime env-override layer wiring, plus the live transport data path. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn hedging_override_env_var_reaches_runtime_and_keeps_reads_healthy( ) -> Result<(), Box> { diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_item_operations.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_item_operations.rs index 72cc71e0133..9df01f634a0 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_item_operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_item_operations.rs @@ -31,8 +31,12 @@ struct TestItem { /// 5. Validates diagnostics for both operations #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn create_and_read_item() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -101,8 +105,12 @@ pub async fn create_and_read_item() -> Result<(), Box> { /// Tests that control plane operations use the metadata pipeline. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn control_plane_uses_metadata_pipeline() -> Result<(), Box> { DriverTestClient::run_with_unique_db(async |context, database| { @@ -143,8 +151,12 @@ pub async fn control_plane_uses_metadata_pipeline() -> Result<(), Box /// Tests diagnostics content for emulator operations. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn diagnostics_contain_expected_fields() -> Result<(), Box> { DriverTestClient::run_with_unique_db(async |context, database| { diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_patch.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_patch.rs index 4c8e60077f4..e26cec75fef 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_patch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_patch.rs @@ -42,8 +42,12 @@ use std::error::Error; /// response body and the subsequent read. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_basic_set() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -103,8 +107,12 @@ pub async fn cosmos_patch_basic_set() -> Result<(), Box> { /// guard with no network call. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_pk_guard() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -178,8 +186,12 @@ pub async fn cosmos_patch_pk_guard() -> Result<(), Box> { /// rejected. Exercises the `MultiHash` branch of `validate_partition_key_paths`. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_pk_guard_hierarchical() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -799,8 +811,12 @@ fn assert_post_image_props(actual: &Value, expected_props: &Value, case_id: &str /// error surfacing) against the live emulator. #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_semantics() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -890,8 +906,12 @@ pub async fn cosmos_patch_semantics() -> Result<(), Box> { /// covers the same branch with a scripted dispatcher). #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_read_missing_item_returns_not_found() -> Result<(), Box> { Box::pin(DriverTestClient::run_with_unique_db( @@ -947,8 +967,12 @@ use std::sync::Arc; #[cfg(feature = "fault_injection")] #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_412_retry() -> Result<(), Box> { let custom_412 = CustomResponseBuilder::new(azure_core::http::StatusCode::PreconditionFailed) @@ -1020,8 +1044,12 @@ pub async fn cosmos_patch_412_retry() -> Result<(), Box> { #[cfg(feature = "fault_injection")] #[tokio::test] #[cfg_attr( - not(any(test_category = "emulator", test_category = "emulator_vnext")), - ignore = "requires test_category 'emulator' or 'emulator_vnext'" + not(any( + test_category = "emulator", + test_category = "emulator_vnext", + test_category = "emulator_inmemory" + )), + ignore = "requires test_category 'emulator', 'emulator_vnext', or 'emulator_inmemory'" )] pub async fn cosmos_patch_412_exhaustion() -> Result<(), Box> { let custom_412 = CustomResponseBuilder::new(azure_core::http::StatusCode::PreconditionFailed) diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs new file mode 100644 index 00000000000..6707a96cb6f --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! CI contract for the out-of-process in-memory emulator host. + +use serde::Deserialize; + +use crate::framework::DriverTestClient; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct HostHealth { + gateway20_enabled: bool, + connectivity_probes: usize, + gateway20_requests: usize, +} + +#[tokio::test] +#[cfg_attr( + not(test_category = "emulator_inmemory"), + ignore = "requires test_category 'emulator_inmemory'" +)] +async fn configured_host_mode_is_exercised() -> Result<(), Box> { + DriverTestClient::run_with_unique_db(async |context, database| { + let container_name = context.unique_container_name(); + let container = context + .create_container(&database, &container_name, "/pk") + .await?; + let body = br#"{"id":"host-mode-item","pk":"pk1","value":42}"#; + context + .create_item(&container, "host-mode-item", "pk1", body) + .await?; + context + .read_item(&container, "host-mode-item", "pk1") + .await?; + Ok(()) + }) + .await?; + + let management_endpoint = std::env::var("AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT")?; + let health_endpoint = url::Url::parse(&management_endpoint)?.join("health")?; + let health = reqwest::Client::new() + .get(health_endpoint) + .send() + .await? + .error_for_status()? + .bytes() + .await?; + let health: HostHealth = serde_json::from_slice(&health)?; + match std::env::var("AZURE_COSMOS_EMULATOR_FLAVOR").as_deref() { + Ok("inmemory-v1") => assert!(!health.gateway20_enabled), + Ok("inmemory-v2") => { + assert!(health.gateway20_enabled); + assert!(health.connectivity_probes > 0); + assert!(health.gateway20_requests > 0); + } + flavor => panic!("unexpected hosted emulator flavor: {flavor:?}"), + } + Ok(()) +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/mod.rs index 57ea58665f4..f1491f5b7cd 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/mod.rs @@ -7,6 +7,7 @@ mod driver_backup_endpoints; mod driver_hedging_kill_switch; mod driver_item_operations; mod driver_patch; +mod hosted_emulator_ci; mod driver_account_metadata_failover; mod driver_fault_injection; diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs index 6290e7ce3e6..3ae6b1d00c0 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs @@ -45,6 +45,29 @@ async fn create_duplicate_409() { assert_eq!(response.status(), StatusCode::Conflict); } +#[tokio::test] +async fn upsert_missing_with_if_match_returns_412() { + let ctx = setup_single_region().await; + let body = serde_json::json!({"id": "missing", "pk": "pk1", "value": 42}); + let mut request = create_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + false, + ); + request + .headers_mut() + .insert(IS_UPSERT.clone(), HeaderValue::from_static("True")); + request + .headers_mut() + .insert(IF_MATCH.clone(), HeaderValue::from_static("\"stale\"")); + + let response = ctx.emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), StatusCode::PreconditionFailed); +} + #[tokio::test] async fn replace_nonexistent_404() { let ctx = setup_single_region().await; diff --git a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml new file mode 100644 index 00000000000..9e8c2b5852b --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "azure_data_cosmos_emulator" +version = "0.1.0" +description = "Out-of-process host for the Azure Cosmos DB in-memory emulator" +publish = false +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +axum.workspace = true +azure_core.workspace = true +azure_data_cosmos_driver = { path = "../azure_data_cosmos_driver", default-features = false, features = [ + "rustls", + "__internal_in_memory_emulator", +] } +clap.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = [ + "fs", + "net", + "rt-multi-thread", + "signal", + "sync", +] } +tracing.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } +url.workspace = true + +[lints] +workspace = true diff --git a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json new file mode 100644 index 00000000000..17bfc7e8c1c --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json @@ -0,0 +1,25 @@ +{ + "account": { + "id": "emulator-account", + "writeMode": "single", + "consistency": "Session", + "perPartitionFailover": false, + "throttling": false, + "regions": [ + { + "name": "East US", + "gatewayPort": 0, + "regionId": 0 + } + ], + "replication": { + "minDelayMs": 0, + "maxDelayMs": 0, + "maxBufferedReplications": 10000 + } + }, + "management": { + "port": 0 + }, + "databases": [] +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json new file mode 100644 index 00000000000..86263f36547 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json @@ -0,0 +1,26 @@ +{ + "account": { + "id": "emulator-account", + "writeMode": "single", + "consistency": "Session", + "perPartitionFailover": false, + "throttling": false, + "regions": [ + { + "name": "East US", + "gatewayPort": 0, + "gateway20Port": 0, + "regionId": 0 + } + ], + "replication": { + "minDelayMs": 0, + "maxDelayMs": 0, + "maxBufferedReplications": 10000 + } + }, + "management": { + "port": 0 + }, + "databases": [] +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 4202bd21af5..276311f7fe8 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -1,6 +1,6 @@ # Hosted In-Memory Cosmos DB Emulator — Plan & Summary -**Status:** Draft / Plan (not yet implemented) +**Status:** PR1 implementation in progress **Date:** 2026-07-14 **Crates:** `azure_data_cosmos_emulator` (new binary host), `azure_data_cosmos_driver` (emulator core, behind a feature) **Feature gate:** `__internal_in_memory_emulator` (non-SemVer emulator surface) @@ -515,13 +515,9 @@ store.set_write_region("West US")?; // runtime single-write failover ### 7.4 Gateway 2.0 server codec (behind host feature) ```rust -// Server-side codec methods extracted and completed from the existing test parsing/building logic. -use azure_data_cosmos_driver::driver::transport::rntbd::{RntbdRequestFrame, RntbdResponse}; - -let frame = RntbdRequestFrame::read(&request_bytes)?; // decode inbound (server side) -// ... dispatch through the store ... -let mut out = Vec::new(); -response.write(&mut out)?; // encode outbound (server side) +// The inverse codec remains co-located with the client codec inside the driver. +// The host consumes one high-level API instead of public token/frame internals. +let response = emulator.execute_gateway_v2_request(&request).await?; ``` --- diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs new file mode 100644 index 00000000000..d07939c49c7 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs @@ -0,0 +1,520 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use std::{net::SocketAddr, path::Path, sync::Arc, time::Duration}; + +use azure_core::http::{headers::HeaderValue, Method, Request}; +use azure_data_cosmos_driver::{ + in_memory_emulator::{ + ConsistencyLevel, ContainerConfig, InMemoryEmulatorHttpClient, ReplicationConfig, + VirtualAccountConfig, VirtualRegion, WriteMode, + }, + models::PartitionKeyDefinition, +}; +use serde::Deserialize; +use tokio::net::TcpListener; +use url::Url; + +use crate::Result; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct EmulatorConfig { + pub(crate) account: AccountConfig, + #[serde(default)] + pub(crate) management: ManagementConfig, + #[serde(default)] + databases: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct AccountConfig { + pub(crate) id: String, + write_mode: WriteModeConfig, + consistency: ConsistencyConfig, + #[serde(default)] + per_partition_failover: bool, + #[serde(default)] + throttling: bool, + regions: Vec, + #[serde(default)] + replication: ReplicationSettings, + #[serde(default)] + replication_overrides: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct RegionConfig { + pub(crate) name: String, + #[serde(default)] + pub(crate) gateway_port: u16, + pub(crate) gateway20_port: Option, + pub(crate) region_id: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ManagementConfig { + #[serde(default)] + pub(crate) port: u16, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct DatabaseConfig { + id: String, + #[serde(default)] + containers: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ContainerSettings { + id: String, + partition_key: PartitionKeyDefinition, + #[serde(default = "default_partition_count")] + partition_count: u32, + throughput: Option, + #[serde(default)] + seed_items: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct SeedItem { + partition_key: Vec, + document: serde_json::Value, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ReplicationSettings { + #[serde(default = "default_min_delay_ms")] + min_delay_ms: u64, + #[serde(default = "default_max_delay_ms")] + max_delay_ms: u64, + #[serde(default = "default_max_buffered_replications")] + max_buffered_replications: usize, +} + +impl Default for ReplicationSettings { + fn default() -> Self { + Self { + min_delay_ms: default_min_delay_ms(), + max_delay_ms: default_max_delay_ms(), + max_buffered_replications: default_max_buffered_replications(), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ReplicationOverride { + source: String, + target: String, + min_delay_ms: u64, + max_delay_ms: u64, + #[serde(default = "default_max_buffered_replications")] + max_buffered_replications: usize, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +enum WriteModeConfig { + Single, + Multi, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +enum ConsistencyConfig { + Strong, + BoundedStaleness, + Session, + ConsistentPrefix, + Eventual, +} + +#[derive(Clone, Debug)] +pub(crate) struct GatewayBinding { + pub(crate) region_name: String, + pub(crate) gateway_url: Url, + pub(crate) gateway20_url: Option, +} + +pub(crate) struct BoundGateway { + pub(crate) binding: GatewayBinding, + pub(crate) gateway_listener: TcpListener, + pub(crate) gateway20_listener: Option, +} + +impl EmulatorConfig { + pub(crate) async fn load(path: &Path) -> Result { + let contents = tokio::fs::read(path).await?; + let config: Self = serde_json::from_slice(&contents)?; + config.validate()?; + Ok(config) + } + + pub(crate) async fn bind_gateways(&self) -> Result> { + let mut gateways = Vec::with_capacity(self.account.regions.len()); + for region in &self.account.regions { + let gateway_listener = + TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], region.gateway_port))).await?; + let gateway_url = loopback_url(gateway_listener.local_addr()?.port())?; + let gateway20_listener = match region.gateway20_port { + Some(port) => { + Some(TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))).await?) + } + None => None, + }; + let gateway20_url = gateway20_listener + .as_ref() + .map(|listener| loopback_url(listener.local_addr()?.port())) + .transpose()?; + gateways.push(BoundGateway { + binding: GatewayBinding { + region_name: region.name.clone(), + gateway_url, + gateway20_url, + }, + gateway_listener, + gateway20_listener, + }); + } + Ok(gateways) + } + + pub(crate) fn create_emulator( + &self, + bindings: &[GatewayBinding], + ) -> Result> { + if bindings.len() != self.account.regions.len() { + return Err("every configured region must have a bound gateway".into()); + } + let regions = self + .account + .regions + .iter() + .zip(bindings) + .map(|(region, binding)| { + let mut virtual_region = + VirtualRegion::new(®ion.name, binding.gateway_url.clone()); + if let Some(gateway20_url) = &binding.gateway20_url { + virtual_region = virtual_region.with_thin_client_url(gateway20_url.clone()); + } + match region.region_id { + Some(region_id) => virtual_region.with_region_id(region_id), + None => virtual_region, + } + }) + .collect(); + + let replication = replication_config( + self.account.replication.min_delay_ms, + self.account.replication.max_delay_ms, + self.account.replication.max_buffered_replications, + )?; + let mut account = VirtualAccountConfig::new(regions)? + .with_account_id(self.account.id.clone()) + .with_write_mode(self.account.write_mode.into()) + .with_consistency(self.account.consistency.into()) + .with_replication_config(replication) + .with_throttling_enabled(self.account.throttling) + .with_per_partition_failover(self.account.per_partition_failover); + + for replication_override in &self.account.replication_overrides { + let override_config = replication_config( + replication_override.min_delay_ms, + replication_override.max_delay_ms, + replication_override.max_buffered_replications, + )?; + account = account.with_replication_override( + &replication_override.source, + &replication_override.target, + override_config, + )?; + } + + Ok(Arc::new(InMemoryEmulatorHttpClient::new(account))) + } + + pub(crate) async fn provision( + &self, + emulator: &Arc, + gateway_url: &Url, + ) -> Result<()> { + let store = emulator.store(); + + for database in &self.databases { + store.create_database(&database.id); + for container in &database.containers { + let mut container_config = + ContainerConfig::new().with_partition_count(container.partition_count); + if let Some(throughput) = container.throughput { + container_config = container_config.with_throughput(throughput); + } + store.create_container_with_config( + &database.id, + &container.id, + container.partition_key.clone(), + container_config.build()?, + ); + + for seed_item in &container.seed_items { + seed_document( + emulator, + gateway_url, + &database.id, + &container.id, + seed_item, + ) + .await?; + } + } + } + Ok(()) + } + + fn validate(&self) -> Result<()> { + if self.account.id.trim().is_empty() { + return Err("account.id must not be empty".into()); + } + if self.account.regions.is_empty() { + return Err("account.regions must contain at least one region".into()); + } + let mut ports = std::collections::HashSet::new(); + if self.management.port != 0 { + ports.insert(self.management.port); + } + let mut region_names = std::collections::HashSet::new(); + let mut region_ids = std::collections::HashSet::new(); + for (index, region) in self.account.regions.iter().enumerate() { + if region.name.trim().is_empty() { + return Err("region names must not be empty".into()); + } + if !region_names.insert(region.name.to_ascii_lowercase()) { + return Err( + format!("region name '{}' is configured more than once", region.name).into(), + ); + } + let region_id = region.region_id.unwrap_or(index as u64); + if !region_ids.insert(region_id) { + return Err(format!("region ID {region_id} is configured more than once").into()); + } + if region.gateway_port != 0 && !ports.insert(region.gateway_port) { + return Err( + format!("port {} is configured more than once", region.gateway_port).into(), + ); + } + if let Some(gateway20_port) = region.gateway20_port { + if gateway20_port != 0 && !ports.insert(gateway20_port) { + return Err( + format!("port {gateway20_port} is configured more than once").into(), + ); + } + } + } + Ok(()) + } +} + +async fn seed_document( + emulator: &InMemoryEmulatorHttpClient, + gateway_url: &Url, + database_id: &str, + container_id: &str, + seed_item: &SeedItem, +) -> Result<()> { + let url = gateway_url.join(&format!("dbs/{database_id}/colls/{container_id}/docs"))?; + let mut request = Request::new(url, Method::Post); + request.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from(serde_json::to_string(&seed_item.partition_key)?), + ); + request.set_body(serde_json::to_vec(&seed_item.document)?); + + let response = emulator.execute_request(&request).await?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + + let raw = response.try_into_raw_response().await?; + Err(format!( + "failed to seed {database_id}/{container_id}: HTTP {}: {}", + u16::from(status), + String::from_utf8_lossy(raw.body().as_ref()) + ) + .into()) +} + +fn replication_config( + min_delay_ms: u64, + max_delay_ms: u64, + max_buffered: usize, +) -> Result { + Ok(ReplicationConfig::range( + Duration::from_millis(min_delay_ms), + Duration::from_millis(max_delay_ms), + )? + .with_max_buffered_replications(max_buffered)) +} + +fn loopback_url(port: u16) -> Result { + Ok(Url::parse(&format!("http://127.0.0.1:{port}/"))?) +} + +fn default_partition_count() -> u32 { + 4 +} + +fn default_min_delay_ms() -> u64 { + 20 +} + +fn default_max_delay_ms() -> u64 { + 50 +} + +fn default_max_buffered_replications() -> usize { + 10_000 +} + +impl From for WriteMode { + fn from(value: WriteModeConfig) -> Self { + match value { + WriteModeConfig::Single => Self::Single, + WriteModeConfig::Multi => Self::Multi, + } + } +} + +impl From for ConsistencyLevel { + fn from(value: ConsistencyConfig) -> Self { + match value { + ConsistencyConfig::Strong => Self::Strong, + ConsistencyConfig::BoundedStaleness => Self::BoundedStaleness, + ConsistencyConfig::Session => Self::Session, + ConsistencyConfig::ConsistentPrefix => Self::ConsistentPrefix, + ConsistencyConfig::Eventual => Self::Eventual, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn parses_and_provisions_seed_items() { + let config: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US", "gatewayPort": 0 }] + }, + "management": { "port": 0 }, + "databases": [{ + "id": "testdb", + "containers": [{ + "id": "testcoll", + "partitionKey": { "paths": ["/pk"], "kind": "Hash", "version": 2 }, + "partitionCount": 1, + "throughput": 400, + "seedItems": [{ + "partitionKey": ["pk1"], + "document": { "id": "item1", "pk": "pk1", "value": 42 } + }] + }] + }] + })) + .unwrap(); + config.validate().unwrap(); + let bound_gateways = config.bind_gateways().await.unwrap(); + let bindings: Vec<_> = bound_gateways + .iter() + .map(|gateway| gateway.binding.clone()) + .collect(); + let gateway_url = bindings[0].gateway_url.clone(); + let emulator = config.create_emulator(&bindings).unwrap(); + config.provision(&emulator, &gateway_url).await.unwrap(); + + let mut request = Request::new( + gateway_url + .join("dbs/testdb/colls/testcoll/docs/item1") + .unwrap(), + Method::Get, + ); + request.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + let response = emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), azure_core::http::StatusCode::Ok); + } + + #[test] + fn rejects_duplicate_ports() { + let config: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US", "gatewayPort": 9090 }] + }, + "management": { "port": 9090 } + })) + .unwrap(); + + assert!(config.validate().is_err()); + } + + #[tokio::test] + async fn zero_ports_use_distinct_os_assigned_endpoints() { + let zero_port: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US", "gateway20Port": 0 }] + } + })) + .unwrap(); + zero_port.validate().unwrap(); + assert_eq!(zero_port.management.port, 0); + + let gateways = zero_port.bind_gateways().await.unwrap(); + let gateway = &gateways[0]; + let gateway_port = gateway.binding.gateway_url.port().unwrap(); + let gateway20_port = gateway + .binding + .gateway20_url + .as_ref() + .unwrap() + .port() + .unwrap(); + assert_ne!(gateway_port, 0); + assert_ne!(gateway20_port, 0); + assert_ne!(gateway_port, gateway20_port); + } + + #[test] + fn rejects_duplicate_region_identity() { + let duplicate: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [ + { "name": "East US", "gatewayPort": 18081, "regionId": 1 }, + { "name": "east us", "gatewayPort": 18082, "regionId": 1 } + ] + } + })) + .unwrap(); + assert!(duplicate.validate().is_err()); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs new file mode 100644 index 00000000000..c1828c2b65f --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use std::{io, sync::Arc}; + +use axum::{ + body::{to_bytes, Body}, + extract::{Request, State}, + http::{HeaderName, HeaderValue, Response, StatusCode}, + response::IntoResponse, + routing::any, + Json, Router, +}; +use azure_core::http::{Method, Request as CosmosRequest}; +use azure_data_cosmos_driver::in_memory_emulator::InMemoryEmulatorHttpClient; +use serde_json::json; +use tokio::net::TcpListener; +use url::Url; + +use crate::config::GatewayBinding; + +const MAX_REQUEST_BODY_SIZE: usize = 16 * 1024 * 1024; + +#[derive(Clone)] +struct GatewayState { + emulator: Arc, + base_url: Url, +} + +pub(crate) async fn serve( + listener: TcpListener, + binding: GatewayBinding, + emulator: Arc, +) -> io::Result<()> { + tracing::info!( + region = binding.region_name, + endpoint = %binding.gateway_url, + "Cosmos gateway listener ready" + ); + let router = router(emulator, binding.gateway_url); + axum::serve(listener, router).await +} + +pub(crate) fn router(emulator: Arc, base_url: Url) -> Router { + Router::new() + .fallback(any(dispatch)) + .with_state(GatewayState { emulator, base_url }) +} + +async fn dispatch(State(state): State, request: Request) -> Response { + match execute(state, request).await { + Ok(response) => response, + Err((status, message)) => (status, Json(json!({ "error": message }))).into_response(), + } +} + +async fn execute( + state: GatewayState, + request: Request, +) -> Result, (StatusCode, String)> { + let cosmos_request = into_cosmos_request(request, &state.base_url).await?; + let response = state + .emulator + .execute_request(&cosmos_request) + .await + .map_err(internal_error)?; + into_http_response(response).await +} + +pub(crate) async fn into_cosmos_request( + request: Request, + base_url: &Url, +) -> Result { + let (parts, body) = request.into_parts(); + let method = parts + .method + .as_str() + .parse::() + .map_err(|error| (StatusCode::METHOD_NOT_ALLOWED, error.to_string()))?; + let path_and_query = parts + .uri + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let url = base_url + .join(path_and_query.trim_start_matches('/')) + .map_err(internal_error)?; + let bytes = to_bytes(body, MAX_REQUEST_BODY_SIZE) + .await + .map_err(|error| (StatusCode::PAYLOAD_TOO_LARGE, error.to_string()))?; + + let mut cosmos_request = CosmosRequest::new(url, method); + for (name, value) in &parts.headers { + let value = value + .to_str() + .map_err(|error| (StatusCode::BAD_REQUEST, error.to_string()))?; + cosmos_request + .headers_mut() + .insert(name.as_str().to_owned(), value.to_owned()); + } + if !bytes.is_empty() { + cosmos_request.set_body(bytes.to_vec()); + } + Ok(cosmos_request) +} + +pub(crate) async fn into_http_response( + response: azure_core::http::AsyncRawResponse, +) -> Result, (StatusCode, String)> { + let response = response + .try_into_raw_response() + .await + .map_err(internal_error)?; + + let mut builder = Response::builder().status(u16::from(response.status())); + for (name, value) in response.headers().iter() { + let name = HeaderName::from_bytes(name.as_str().as_bytes()).map_err(internal_error)?; + let value = HeaderValue::from_str(value.as_str()).map_err(internal_error)?; + builder = builder.header(name, value); + } + builder + .body(Body::from(response.body().as_ref().to_vec())) + .map_err(internal_error) +} + +pub(crate) fn internal_error(error: impl std::fmt::Display) -> (StatusCode, String) { + (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use azure_data_cosmos_driver::in_memory_emulator::{VirtualAccountConfig, VirtualRegion}; + + #[tokio::test] + async fn account_read_flows_through_http_bridge() { + let base_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let config = + VirtualAccountConfig::new(vec![VirtualRegion::new("East US", base_url.clone())]) + .unwrap(); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(config)); + let request = Request::builder().uri("/").body(Body::empty()).unwrap(); + + let response = execute(GatewayState { emulator, base_url }, request) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs new file mode 100644 index 00000000000..1446e887ec6 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs @@ -0,0 +1,272 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use std::{io, sync::Arc}; + +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; + +use axum::{ + body::Body, + extract::{Request, State}, + http::{Response, StatusCode, Version}, + response::IntoResponse, + routing::any, + Json, Router, +}; +use azure_data_cosmos_driver::in_memory_emulator::InMemoryEmulatorHttpClient; +use serde_json::json; +use tokio::net::TcpListener; +use url::Url; + +use crate::{config::GatewayBinding, data_plane, metrics::HostMetrics}; + +#[derive(Clone)] +struct GatewayV2State { + emulator: Arc, + base_url: Url, + metrics: Arc, + #[cfg(test)] + request_count: Option>, + #[cfg(test)] + probe_count: Option>, +} + +pub(crate) async fn serve( + listener: TcpListener, + binding: GatewayBinding, + emulator: Arc, + metrics: Arc, +) -> io::Result<()> { + let Some(base_url) = binding.gateway20_url else { + return Ok(()); + }; + tracing::info!( + region = binding.region_name, + endpoint = %base_url, + "Cosmos Gateway 2.0 listener ready" + ); + axum::serve(listener, router(emulator, base_url, metrics)).await +} + +fn router( + emulator: Arc, + base_url: Url, + metrics: Arc, +) -> Router { + Router::new() + .fallback(any(dispatch)) + .with_state(GatewayV2State { + emulator, + base_url, + metrics, + #[cfg(test)] + request_count: None, + #[cfg(test)] + probe_count: None, + }) +} + +async fn dispatch(State(state): State, request: Request) -> Response { + match execute(state, request).await { + Ok(response) => response, + Err((status, message)) => { + tracing::error!(status = %status, error = %message, "Gateway 2.0 request failed"); + (status, Json(json!({ "error": message }))).into_response() + } + } +} + +async fn execute( + state: GatewayV2State, + request: Request, +) -> Result, (StatusCode, String)> { + if request.version() != Version::HTTP_2 { + return Err(( + StatusCode::HTTP_VERSION_NOT_SUPPORTED, + "Gateway 2.0 requires HTTP/2".to_owned(), + )); + } + if request.method() == axum::http::Method::POST && request.uri().path() == "/connectivity-probe" + { + state.metrics.record_connectivity_probe(); + #[cfg(test)] + if let Some(probe_count) = &state.probe_count { + probe_count.fetch_add(1, Ordering::SeqCst); + } + return Response::builder() + .status(StatusCode::OK) + .body(Body::empty()) + .map_err(data_plane::internal_error); + } + state.metrics.record_gateway20_request(); + #[cfg(test)] + if let Some(request_count) = &state.request_count { + request_count.fetch_add(1, Ordering::SeqCst); + } + + let request = data_plane::into_cosmos_request(request, &state.base_url).await?; + let response = state + .emulator + .execute_gateway_v2_request(&request) + .await + .map_err(data_plane::internal_error)?; + data_plane::into_http_response(response).await +} + +#[cfg(test)] +mod tests { + use super::*; + use azure_core::http::{headers::HeaderValue, Method, Request as CosmosRequest}; + use azure_data_cosmos_driver::{ + driver::CosmosDriverRuntime, + in_memory_emulator::{ContainerConfig, VirtualAccountConfig, VirtualRegion}, + models::{ + AccountReference, CosmosOperation, ItemReference, PartitionKey, PartitionKeyDefinition, + }, + options::{DriverOptions, OperationOptions}, + }; + + #[tokio::test] + async fn connectivity_probe_requires_http2_and_returns_ok() { + let base_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", Url::parse("http://127.0.0.1:18081/").unwrap()) + .with_thin_client_url(base_url.clone()); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new( + VirtualAccountConfig::new(vec![region]).unwrap(), + )); + let state = GatewayV2State { + emulator, + base_url, + metrics: Arc::new(HostMetrics::default()), + request_count: None, + probe_count: None, + }; + + let mut request = Request::builder() + .method("POST") + .uri("/connectivity-probe") + .body(Body::empty()) + .unwrap(); + *request.version_mut() = Version::HTTP_2; + let response = execute(state.clone(), request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let request = Request::builder() + .method("POST") + .uri("/connectivity-probe") + .body(Body::empty()) + .unwrap(); + let error = execute(state, request).await.unwrap_err(); + assert_eq!(error.0, StatusCode::HTTP_VERSION_NOT_SUPPORTED); + } + + #[tokio::test] + async fn real_driver_uses_hosted_gateway20_over_h2c() { + let gateway_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let thin_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let gateway_url = Url::parse(&format!( + "http://{}/", + gateway_listener.local_addr().unwrap() + )) + .unwrap(); + let thin_url = + Url::parse(&format!("http://{}/", thin_listener.local_addr().unwrap())).unwrap(); + let region = VirtualRegion::new("East US", gateway_url.clone()) + .with_thin_client_url(thin_url.clone()); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new( + VirtualAccountConfig::new(vec![region]).unwrap(), + )); + let store = emulator.store(); + store.create_database("db"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + store.create_container_with_config( + "db", + "coll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + let mut seed = CosmosRequest::new( + gateway_url.join("dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + seed.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + seed.set_body( + serde_json::to_vec(&serde_json::json!({ + "id": "item1", "pk": "pk1", "value": 42 + })) + .unwrap(), + ); + assert!(emulator + .execute_request(&seed) + .await + .unwrap() + .status() + .is_success()); + + let gateway_router = data_plane::router(emulator.clone(), gateway_url.clone()); + let gateway_task = + tokio::spawn(async move { axum::serve(gateway_listener, gateway_router).await }); + let request_count = Arc::new(AtomicUsize::new(0)); + let probe_count = Arc::new(AtomicUsize::new(0)); + let thin_state = GatewayV2State { + emulator, + base_url: thin_url, + metrics: Arc::new(HostMetrics::default()), + request_count: Some(request_count.clone()), + probe_count: Some(probe_count.clone()), + }; + let thin_task = tokio::spawn(async move { + axum::serve( + thin_listener, + Router::new().fallback(any(dispatch)).with_state(thin_state), + ) + .await + }); + + let runtime = CosmosDriverRuntime::builder().build().await.unwrap(); + let account = AccountReference::with_master_key( + gateway_url, + "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + ); + let driver = runtime + .create_driver(DriverOptions::builder(account).build()) + .await + .unwrap(); + let container = driver.resolve_container("db", "coll").await.unwrap(); + let item = + ItemReference::from_name(&container, PartitionKey::from("pk1"), "item1".to_owned()); + let response = driver + .execute_singleton_operation( + CosmosOperation::read_item(item), + OperationOptions::default(), + ) + .await + .unwrap(); + + assert_eq!( + response.status().status_code(), + azure_core::http::StatusCode::Ok + ); + assert!( + probe_count.load(Ordering::SeqCst) > 0, + "the driver must probe the advertised Gateway 2.0 endpoint" + ); + assert!( + request_count.load(Ordering::SeqCst) > 0, + "the point read must reach the Gateway 2.0 listener" + ); + + gateway_task.abort(); + thin_task.abort(); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs new file mode 100644 index 00000000000..f7b540accbf --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +mod config; +mod data_plane; +mod gateway_v2; +mod management; +mod metrics; + +use std::{ + io::{self, Write}, + net::SocketAddr, + path::PathBuf, +}; + +use clap::Parser; +use config::{EmulatorConfig, GatewayBinding}; +use metrics::HostMetrics; +use serde::Serialize; +use tokio::net::TcpListener; +use url::Url; + +type Result = std::result::Result>; + +#[derive(Debug, Parser)] +#[command(about = "Hosts the Azure Cosmos DB in-memory emulator over HTTP")] +struct Args { + /// JSON configuration file describing the virtual account and seed data. + #[arg(long)] + config: PathBuf, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_writer(io::stderr) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "azure_data_cosmos_emulator=info".into()), + ) + .init(); + + let args = Args::parse(); + let config = EmulatorConfig::load(&args.config).await?; + let bound_gateways = config.bind_gateways().await?; + let bindings: Vec<_> = bound_gateways + .iter() + .map(|gateway| gateway.binding.clone()) + .collect(); + let management_listener = + TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], config.management.port))).await?; + let management_endpoint = loopback_url(management_listener.local_addr()?.port())?; + let account_endpoint = bindings + .first() + .ok_or("no emulator gateway listeners were configured")? + .gateway_url + .clone(); + let emulator = config.create_emulator(&bindings)?; + config.provision(&emulator, &account_endpoint).await?; + let metrics = std::sync::Arc::new(HostMetrics::default()); + + let mut listeners = tokio::task::JoinSet::new(); + for bound_gateway in bound_gateways { + let binding = bound_gateway.binding; + let gateway_emulator = emulator.clone(); + let gateway_binding = binding.clone(); + listeners.spawn(async move { + data_plane::serve( + bound_gateway.gateway_listener, + gateway_binding, + gateway_emulator, + ) + .await + }); + if let Some(gateway20_listener) = bound_gateway.gateway20_listener { + let gateway20_emulator = emulator.clone(); + let metrics = metrics.clone(); + listeners.spawn(async move { + gateway_v2::serve(gateway20_listener, binding, gateway20_emulator, metrics).await + }); + } + } + let account_id = config.account.id.clone(); + let management_bindings = bindings.clone(); + listeners.spawn(async move { + management::serve( + management_listener, + emulator, + account_id, + management_bindings, + metrics, + ) + .await + }); + write_ready_record(management_endpoint, account_endpoint, &bindings)?; + + match listeners.join_next().await { + Some(Ok(Ok(()))) => Err("an emulator listener stopped unexpectedly".into()), + Some(Ok(Err(error))) => Err(error.into()), + Some(Err(error)) => Err(error.into()), + None => Err("no emulator listeners were configured".into()), + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ReadyRecord { + event: &'static str, + management_endpoint: String, + account_endpoint: String, + regions: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ReadyRegion { + name: String, + gateway_endpoint: String, + #[serde(skip_serializing_if = "Option::is_none")] + gateway20_endpoint: Option, +} + +fn write_ready_record( + management_endpoint: Url, + account_endpoint: Url, + bindings: &[GatewayBinding], +) -> Result<()> { + let record = ReadyRecord { + event: "ready", + management_endpoint: management_endpoint.into(), + account_endpoint: account_endpoint.into(), + regions: bindings + .iter() + .map(|binding| ReadyRegion { + name: binding.region_name.clone(), + gateway_endpoint: binding.gateway_url.as_str().to_owned(), + gateway20_endpoint: binding + .gateway20_url + .as_ref() + .map(|url| url.as_str().to_owned()), + }) + .collect(), + }; + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + serde_json::to_writer(&mut stdout, &record)?; + writeln!(stdout)?; + stdout.flush()?; + Ok(()) +} + +fn loopback_url(port: u16) -> Result { + Ok(Url::parse(&format!("http://127.0.0.1:{port}/"))?) +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs new file mode 100644 index 00000000000..050371d156e --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs @@ -0,0 +1,1004 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// cspell:ignore hexdigit + +use std::{ + collections::HashMap, + io, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; + +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post, put}, + Json, Router, +}; +use azure_data_cosmos_driver::in_memory_emulator::{ + EmulatorStore, Epk, InMemoryEmulatorHttpClient, ManualControlPlaneOperation, WriteMode, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tokio::{net::TcpListener, sync::Mutex}; + +use crate::{config::GatewayBinding, metrics::HostMetrics}; + +const MAX_CONTROL_PLANE_LOCK_DURATION_MS: u64 = 60_000; + +#[derive(Clone)] +struct ManagementState { + emulator: Arc, + account_id: Arc, + bindings: Arc<[GatewayBinding]>, + metrics: Arc, + operations: Arc, +} + +#[derive(Default)] +struct OperationRegistry { + next_id: AtomicU64, + records: Mutex>, +} + +pub(crate) async fn serve( + listener: TcpListener, + emulator: Arc, + account_id: String, + bindings: Vec, + metrics: Arc, +) -> io::Result<()> { + tracing::info!(endpoint = %listener.local_addr()?, "emulator management API ready"); + axum::serve(listener, router(emulator, account_id, bindings, metrics)).await +} + +fn router( + emulator: Arc, + account_id: String, + bindings: Vec, + metrics: Arc, +) -> Router { + let state = ManagementState { + emulator, + account_id: account_id.into(), + bindings: bindings.into(), + metrics, + operations: Arc::new(OperationRegistry::default()), + }; + Router::new() + .route("/health", get(health)) + .route("/account", get(account)) + .route( + "/databases/{database}/containers/{container}/partitions/{partition_id}/split", + post(split_partition), + ) + .route( + "/databases/{database}/containers/{container}/partitions/merge", + post(merge_partitions), + ) + .route("/operations/{operation_id}", get(get_operation)) + .route( + "/operations/{operation_id}/advance", + post(advance_operation), + ) + .route( + "/config/per-partition-failover", + put(set_per_partition_failover), + ) + .route( + "/regions/{region}/replication/pause", + post(pause_replication), + ) + .route( + "/regions/{region}/replication/resume", + post(resume_replication), + ) + .with_state(state) +} + +async fn health(State(state): State) -> Json { + Json(json!({ + "status": "ok", + "gateway20Enabled": state + .bindings + .iter() + .any(|binding| binding.gateway20_url.is_some()), + "connectivityProbes": state.metrics.connectivity_probes(), + "gateway20Requests": state.metrics.gateway20_requests() + })) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct AccountResponse { + id: String, + write_mode: &'static str, + consistency: String, + per_partition_failover: bool, + regions: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct RegionResponse { + name: String, + gateway_endpoint: String, + #[serde(skip_serializing_if = "Option::is_none")] + gateway20_endpoint: Option, +} + +async fn account(State(state): State) -> Json { + let config = state.emulator.store().config().clone(); + let write_mode = match config.write_mode() { + WriteMode::Single => "single", + WriteMode::Multi => "multi", + }; + let regions = state + .bindings + .iter() + .map(|binding| RegionResponse { + name: binding.region_name.clone(), + gateway_endpoint: binding.gateway_url.as_str().to_owned(), + gateway20_endpoint: binding + .gateway20_url + .as_ref() + .map(|url| url.as_str().to_owned()), + }) + .collect(); + Json(AccountResponse { + id: state.account_id.to_string(), + write_mode, + consistency: config.consistency().as_str().to_owned(), + per_partition_failover: config.per_partition_failover_enabled(), + regions, + }) +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +enum ProgressionMode { + #[default] + Automatic, + Manual, +} + +#[derive(Clone, Copy, Debug, Serialize)] +enum OperationStatus { + Running, + Succeeded, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +enum OperationPhase { + Preparing, + Swapping, + Succeeded, + Failed, +} + +struct OperationRecord { + status: OperationStatus, + phase: OperationPhase, + result: Option, + error: Option, + action: Option, +} + +enum OperationAction { + PendingSplit { + database: String, + container: String, + parent: u32, + mode: SplitMode, + split_epk: Epk, + }, + ActiveSplit { + database: String, + container: String, + parent: u32, + mode: SplitMode, + split_epk: Epk, + operation: ManualControlPlaneOperation, + }, + PendingMerge { + database: String, + container: String, + partitions: [u32; 2], + }, + ActiveMerge { + database: String, + container: String, + partitions: [u32; 2], + operation: ManualControlPlaneOperation, + }, +} + +impl OperationRegistry { + fn next_id(&self, kind: &str) -> String { + let sequence = self.next_id.fetch_add(1, Ordering::Relaxed) + 1; + format!("op-{kind}-{sequence}") + } +} + +impl OperationRecord { + fn running(action: Option) -> Self { + Self { + status: OperationStatus::Running, + phase: OperationPhase::Preparing, + result: None, + error: None, + action, + } + } + + fn succeed(&mut self, result: serde_json::Value) { + self.status = OperationStatus::Succeeded; + self.phase = OperationPhase::Succeeded; + self.result = Some(result); + self.error = None; + self.action = None; + } + + fn fail(&mut self, error: impl Into) { + self.status = OperationStatus::Failed; + self.phase = OperationPhase::Failed; + self.result = None; + self.error = Some(error.into()); + self.action = None; + } +} + +fn operation_response(operation_id: &str, operation: &OperationRecord) -> serde_json::Value { + let mut response = json!({ + "operationId": operation_id, + "status": operation.status, + "phase": operation.phase, + }); + let object = response + .as_object_mut() + .expect("operation response must be an object"); + if let Some(serde_json::Value::Object(result)) = &operation.result { + object.extend(result.clone()); + } + if let Some(error) = &operation.error { + object.insert("error".to_owned(), error.clone().into()); + } + response +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +enum SplitMode { + #[default] + Midpoint, + Epk, + Storage, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct SplitRequest { + #[serde(default)] + mode: SplitMode, + epk: Option, + #[serde(default)] + progression_mode: ProgressionMode, + lock_duration_ms: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct SplitResponse { + database: String, + container: String, + parent: u32, + children: Vec, + mode: SplitMode, + split_epk: String, +} + +struct SplitOperationDetails { + database: String, + container: String, + parent: u32, + mode: SplitMode, + split_epk: Epk, +} + +async fn split_partition( + State(state): State, + Path((database, container, partition_id)): Path<(String, String, u32)>, + request: Option>, +) -> ApiResult<(StatusCode, Json)> { + let request = request.map(|request| request.0).unwrap_or_default(); + let lock_duration = validate_progression(request.progression_mode, request.lock_duration_ms)?; + let store = state.emulator.store(); + let split_epk = match request.mode { + SplitMode::Midpoint => store.midpoint_split_epk(&database, &container, partition_id)?, + SplitMode::Epk => { + let value = request + .epk + .as_deref() + .ok_or_else(|| ApiError::bad_request("epk is required when mode is 'epk'"))?; + parse_epk(value)? + } + SplitMode::Storage => store.storage_split_epk(&database, &container, partition_id)?, + }; + let operation_id = state.operations.next_id("split"); + let action = match request.progression_mode { + ProgressionMode::Automatic => None, + ProgressionMode::Manual => Some(OperationAction::PendingSplit { + database: database.clone(), + container: container.clone(), + parent: partition_id, + mode: request.mode, + split_epk: split_epk.clone(), + }), + }; + let operation = OperationRecord::running(action); + let response = operation_response(&operation_id, &operation); + state + .operations + .records + .lock() + .await + .insert(operation_id.clone(), operation); + + if request.progression_mode == ProgressionMode::Automatic { + spawn_automatic_split( + state, + operation_id, + SplitOperationDetails { + database, + container, + parent: partition_id, + mode: request.mode, + split_epk, + }, + lock_duration, + ); + } + + Ok((StatusCode::ACCEPTED, Json(response))) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct MergeRequest { + partition_ids: [u32; 2], + #[serde(default)] + progression_mode: ProgressionMode, + lock_duration_ms: Option, +} + +#[derive(Debug, Serialize)] +struct MergeResponse { + merged: [u32; 2], + into: u32, +} + +async fn merge_partitions( + State(state): State, + Path((database, container)): Path<(String, String)>, + Json(request): Json, +) -> ApiResult<(StatusCode, Json)> { + let lock_duration = validate_progression(request.progression_mode, request.lock_duration_ms)?; + let operation_id = state.operations.next_id("merge"); + let action = match request.progression_mode { + ProgressionMode::Automatic => None, + ProgressionMode::Manual => Some(OperationAction::PendingMerge { + database: database.clone(), + container: container.clone(), + partitions: request.partition_ids, + }), + }; + let operation = OperationRecord::running(action); + let response = operation_response(&operation_id, &operation); + state + .operations + .records + .lock() + .await + .insert(operation_id.clone(), operation); + + if request.progression_mode == ProgressionMode::Automatic { + spawn_automatic_merge( + state, + operation_id, + database, + container, + request.partition_ids, + lock_duration, + ); + } + + Ok((StatusCode::ACCEPTED, Json(response))) +} + +fn spawn_automatic_split( + state: ManagementState, + operation_id: String, + details: SplitOperationDetails, + lock_duration: Duration, +) { + tokio::spawn(async move { + let SplitOperationDetails { + database, + container, + parent, + mode, + split_epk, + } = details; + let store = state.emulator.store(); + let operation = + store.begin_manual_split_partition(&database, &container, parent, split_epk.clone()); + set_operation_swapping(&state.operations, &operation_id).await; + if !lock_duration.is_zero() { + tokio::time::sleep(lock_duration).await; + } + let outcome = match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &[parent]); + if children.len() == 2 { + Ok(json!(SplitResponse { + database, + container, + parent, + children, + mode, + split_epk: split_epk.to_hex(), + })) + } else { + Err("split completed without producing exactly two child partitions".to_owned()) + } + } + Err(error) => Err(error.to_string()), + }; + finish_operation(&state.operations, &operation_id, outcome).await; + }); +} + +fn spawn_automatic_merge( + state: ManagementState, + operation_id: String, + database: String, + container: String, + partitions: [u32; 2], + lock_duration: Duration, +) { + tokio::spawn(async move { + let store = state.emulator.store(); + let operation = store.begin_manual_merge_partitions( + &database, + &container, + partitions[0], + partitions[1], + ); + set_operation_swapping(&state.operations, &operation_id).await; + if !lock_duration.is_zero() { + tokio::time::sleep(lock_duration).await; + } + let outcome = match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &partitions); + match children.as_slice() { + [merged_child] => Ok(json!(MergeResponse { + merged: partitions, + into: *merged_child, + })), + _ => Err( + "merge completed without producing exactly one child partition".to_owned(), + ), + } + } + Err(error) => Err(error.to_string()), + }; + finish_operation(&state.operations, &operation_id, outcome).await; + }); +} + +async fn set_operation_swapping(operations: &OperationRegistry, operation_id: &str) { + if let Some(operation) = operations.records.lock().await.get_mut(operation_id) { + operation.phase = OperationPhase::Swapping; + } +} + +async fn finish_operation( + operations: &OperationRegistry, + operation_id: &str, + outcome: Result, +) { + if let Some(operation) = operations.records.lock().await.get_mut(operation_id) { + match outcome { + Ok(result) => operation.succeed(result), + Err(error) => operation.fail(error), + } + } +} + +async fn get_operation( + State(state): State, + Path(operation_id): Path, +) -> ApiResult> { + let operations = state.operations.records.lock().await; + let operation = operations + .get(&operation_id) + .ok_or_else(|| ApiError::not_found(format!("operation '{operation_id}' does not exist")))?; + Ok(Json(operation_response(&operation_id, operation))) +} + +enum ManualCompletion { + Split { + database: String, + container: String, + parent: u32, + mode: SplitMode, + split_epk: Epk, + operation: ManualControlPlaneOperation, + }, + Merge { + database: String, + container: String, + partitions: [u32; 2], + operation: ManualControlPlaneOperation, + }, +} + +async fn advance_operation( + State(state): State, + Path(operation_id): Path, +) -> ApiResult> { + let completion = { + let mut operations = state.operations.records.lock().await; + let operation = operations.get_mut(&operation_id).ok_or_else(|| { + ApiError::not_found(format!("operation '{operation_id}' does not exist")) + })?; + if operation.phase == OperationPhase::Succeeded || operation.phase == OperationPhase::Failed + { + return Err(ApiError::conflict(format!( + "operation '{operation_id}' is already terminal" + ))); + } + let action = operation.action.take().ok_or_else(|| { + ApiError::conflict(format!( + "operation '{operation_id}' is automatic or already advancing" + )) + })?; + match action { + OperationAction::PendingSplit { + database, + container, + parent, + mode, + split_epk, + } => { + let manual_operation = state.emulator.store().begin_manual_split_partition( + &database, + &container, + parent, + split_epk.clone(), + ); + operation.phase = OperationPhase::Swapping; + operation.action = Some(OperationAction::ActiveSplit { + database, + container, + parent, + mode, + split_epk, + operation: manual_operation, + }); + return Ok(Json(operation_response(&operation_id, operation))); + } + OperationAction::PendingMerge { + database, + container, + partitions, + } => { + let manual_operation = state.emulator.store().begin_manual_merge_partitions( + &database, + &container, + partitions[0], + partitions[1], + ); + operation.phase = OperationPhase::Swapping; + operation.action = Some(OperationAction::ActiveMerge { + database, + container, + partitions, + operation: manual_operation, + }); + return Ok(Json(operation_response(&operation_id, operation))); + } + OperationAction::ActiveSplit { + database, + container, + parent, + mode, + split_epk, + operation, + } => ManualCompletion::Split { + database, + container, + parent, + mode, + split_epk, + operation, + }, + OperationAction::ActiveMerge { + database, + container, + partitions, + operation, + } => ManualCompletion::Merge { + database, + container, + partitions, + operation, + }, + } + }; + + let store = state.emulator.store(); + let outcome = match completion { + ManualCompletion::Split { + database, + container, + parent, + mode, + split_epk, + operation, + } => match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &[parent]); + if children.len() == 2 { + Ok(json!(SplitResponse { + database, + container, + parent, + children, + mode, + split_epk: split_epk.to_hex(), + })) + } else { + Err("split completed without producing exactly two child partitions".to_owned()) + } + } + Err(error) => Err(error.to_string()), + }, + ManualCompletion::Merge { + database, + container, + partitions, + operation, + } => match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &partitions); + match children.as_slice() { + [merged_child] => Ok(json!(MergeResponse { + merged: partitions, + into: *merged_child, + })), + _ => Err( + "merge completed without producing exactly one child partition".to_owned(), + ), + } + } + Err(error) => Err(error.to_string()), + }, + }; + finish_operation(&state.operations, &operation_id, outcome).await; + + let operations = state.operations.records.lock().await; + let operation = operations + .get(&operation_id) + .ok_or_else(|| ApiError::not_found(format!("operation '{operation_id}' does not exist")))?; + Ok(Json(operation_response(&operation_id, operation))) +} + +#[derive(Debug, Deserialize, Serialize)] +struct EnabledRequest { + enabled: bool, +} + +async fn set_per_partition_failover( + State(state): State, + Json(request): Json, +) -> Json { + state + .emulator + .store() + .config() + .set_per_partition_failover(request.enabled); + Json(request) +} + +async fn pause_replication( + State(state): State, + Path(region): Path, +) -> ApiResult> { + ensure_region(&state.emulator.store(), ®ion)?; + state.emulator.store().pause_replication(®ion); + Ok(Json(json!({ "region": region, "replication": "paused" }))) +} + +async fn resume_replication( + State(state): State, + Path(region): Path, +) -> ApiResult> { + ensure_region(&state.emulator.store(), ®ion)?; + state.emulator.store().resume_replication(®ion); + Ok(Json(json!({ "region": region, "replication": "resumed" }))) +} + +fn ensure_region(store: &EmulatorStore, name: &str) -> ApiResult<()> { + if store + .config() + .regions() + .iter() + .any(|region| region.name() == name) + { + Ok(()) + } else { + Err(ApiError::not_found(format!( + "region '{name}' is not configured" + ))) + } +} + +fn parse_epk(value: &str) -> ApiResult { + if value.is_empty() + || !value.len().is_multiple_of(2) + || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(ApiError::bad_request( + "epk must be a non-empty, even-length hexadecimal string", + )); + } + Ok(Epk::from(value)) +} + +fn validate_progression( + progression_mode: ProgressionMode, + lock_duration_ms: Option, +) -> ApiResult { + if progression_mode == ProgressionMode::Manual && lock_duration_ms.is_some() { + return Err(ApiError::bad_request( + "lockDurationMs is accepted only when progressionMode is 'automatic'", + )); + } + let lock_duration_ms = lock_duration_ms.unwrap_or_default(); + if lock_duration_ms > MAX_CONTROL_PLANE_LOCK_DURATION_MS { + return Err(ApiError::bad_request(format!( + "lockDurationMs must be <= {MAX_CONTROL_PLANE_LOCK_DURATION_MS}" + ))); + } + Ok(Duration::from_millis(lock_duration_ms)) +} + +type ApiResult = std::result::Result; + +#[derive(Debug)] +struct ApiError { + status: StatusCode, + message: String, +} + +impl ApiError { + fn bad_request(message: impl Into) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: message.into(), + } + } + + fn not_found(message: impl Into) -> Self { + Self { + status: StatusCode::NOT_FOUND, + message: message.into(), + } + } + + fn conflict(message: impl Into) -> Self { + Self { + status: StatusCode::CONFLICT, + message: message.into(), + } + } +} + +impl From for ApiError { + fn from(error: azure_data_cosmos_driver::error::CosmosError) -> Self { + let status = StatusCode::from_u16(u16::from(error.status().status_code())) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Self { + status, + message: error.to_string(), + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + (self.status, Json(json!({ "error": self.message }))).into_response() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use azure_core::http::{headers::HeaderValue, Method, Request}; + use azure_data_cosmos_driver::{ + in_memory_emulator::{ContainerConfig, VirtualAccountConfig, VirtualRegion}, + models::PartitionKeyDefinition, + }; + use url::Url; + + #[tokio::test] + async fn manual_split_and_merge_advance_through_operation_phases() { + let gateway_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let account = + VirtualAccountConfig::new(vec![VirtualRegion::new("East US", gateway_url.clone())]) + .unwrap(); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(account)); + let store = emulator.store(); + store.create_database("testdb"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + store.create_container_with_config( + "testdb", + "testcoll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + for (id, pk, value) in [("1", "a", "small"), ("2", "z", "much-larger-value")] { + let mut request = Request::new( + gateway_url.join("dbs/testdb/colls/testcoll/docs").unwrap(), + Method::Post, + ); + request.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from(format!(r#"["{pk}"]"#)), + ); + request.set_body( + serde_json::to_vec(&serde_json::json!({ + "id": id, "pk": pk, "value": value + })) + .unwrap(), + ); + let response = emulator.execute_request(&request).await.unwrap(); + assert!(response.status().is_success()); + } + + let state = ManagementState { + emulator, + account_id: "test-account".into(), + bindings: Vec::::new().into(), + metrics: Arc::new(HostMetrics::default()), + operations: Arc::new(OperationRegistry::default()), + }; + let response = split_partition( + State(state.clone()), + Path(("testdb".to_owned(), "testcoll".to_owned(), 0)), + Some(Json(SplitRequest { + mode: SplitMode::Storage, + epk: None, + progression_mode: ProgressionMode::Manual, + lock_duration_ms: None, + })), + ) + .await + .unwrap(); + + assert_eq!(response.0, StatusCode::ACCEPTED); + assert_eq!(response.1["phase"], "Preparing"); + let operation_id = response.1["operationId"].as_str().unwrap().to_owned(); + + let swapping = advance_operation(State(state.clone()), Path(operation_id.clone())) + .await + .unwrap(); + assert_eq!(swapping["phase"], "Swapping"); + assert!(state + .emulator + .store() + .child_partition_ids("testdb", "testcoll", &[0]) + .is_empty()); + + let succeeded = advance_operation(State(state.clone()), Path(operation_id.clone())) + .await + .unwrap(); + assert_eq!(succeeded["status"], "Succeeded"); + assert_eq!(succeeded["phase"], "Succeeded"); + assert_eq!(succeeded["children"].as_array().unwrap().len(), 2); + assert!(!succeeded["splitEpk"].as_str().unwrap().is_empty()); + + let queried = get_operation(State(state.clone()), Path(operation_id)) + .await + .unwrap(); + assert_eq!(queried["phase"], "Succeeded"); + + let repeated = split_partition( + State(state.clone()), + Path(("testdb".to_owned(), "testcoll".to_owned(), 0)), + Some(Json(SplitRequest { + mode: SplitMode::Epk, + epk: Some(succeeded["splitEpk"].as_str().unwrap().to_owned()), + progression_mode: ProgressionMode::Automatic, + lock_duration_ms: None, + })), + ) + .await + .unwrap(); + let repeated_id = repeated.1["operationId"].as_str().unwrap().to_owned(); + let mut repeated_phase = None; + for _ in 0..100 { + let operation = get_operation(State(state.clone()), Path(repeated_id.clone())) + .await + .unwrap(); + repeated_phase = operation["phase"].as_str().map(str::to_owned); + if repeated_phase.as_deref() == Some("Failed") { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(repeated_phase.as_deref(), Some("Failed")); + + let children = succeeded["children"] + .as_array() + .unwrap() + .iter() + .map(|child| child.as_u64().unwrap() as u32) + .collect::>(); + let merge = merge_partitions( + State(state.clone()), + Path(("testdb".to_owned(), "testcoll".to_owned())), + Json(MergeRequest { + partition_ids: [children[0], children[1]], + progression_mode: ProgressionMode::Manual, + lock_duration_ms: None, + }), + ) + .await + .unwrap(); + let merge_id = merge.1["operationId"].as_str().unwrap().to_owned(); + let merge_swapping = advance_operation(State(state.clone()), Path(merge_id.clone())) + .await + .unwrap(); + assert_eq!(merge_swapping["phase"], "Swapping"); + let merge_succeeded = advance_operation(State(state.clone()), Path(merge_id)) + .await + .unwrap(); + assert_eq!(merge_succeeded["phase"], "Succeeded"); + assert_eq!(merge_succeeded["merged"].as_array().unwrap().len(), 2); + + let invalid_duration = split_partition( + State(state), + Path(("testdb".to_owned(), "testcoll".to_owned(), 3)), + Some(Json(SplitRequest { + mode: SplitMode::Midpoint, + epk: None, + progression_mode: ProgressionMode::Manual, + lock_duration_ms: Some(0), + })), + ) + .await + .unwrap_err(); + assert_eq!(invalid_duration.status, StatusCode::BAD_REQUEST); + } + + #[test] + fn custom_epk_rejects_malformed_hex() { + assert!(parse_epk("ABC").is_err()); + assert!(parse_epk("not-hex").is_err()); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/metrics.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/metrics.rs new file mode 100644 index 00000000000..b763b9756df --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/metrics.rs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug, Default)] +pub(crate) struct HostMetrics { + connectivity_probes: AtomicUsize, + gateway20_requests: AtomicUsize, +} + +impl HostMetrics { + pub(crate) fn record_connectivity_probe(&self) { + self.connectivity_probes.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn record_gateway20_request(&self) { + self.gateway20_requests.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn connectivity_probes(&self) -> usize { + self.connectivity_probes.load(Ordering::Relaxed) + } + + pub(crate) fn gateway20_requests(&self) -> usize { + self.gateway20_requests.load(Ordering::Relaxed) + } +} diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index f98d3db02c1..3f436706162 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -78,3 +78,9 @@ extends: Path: sdk/cosmos/vnext-emulator-matrix.json Selection: all GenerateVMJobs: true + - Name: Cosmos_inmemory_emulator + # Both jobs use port-zero host configs. Test setup parses the host's + # ready record and exports its OS-assigned account and management URLs. + Path: sdk/cosmos/inmemory-emulator-matrix.json + Selection: all + GenerateVMJobs: true diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 index eebb8b90d23..5ae98f3fefd 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 @@ -7,7 +7,32 @@ $ShutdownTimeout = 30 -if ($IsWindows) { +if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { + if ($env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 -eq 'true') { + try { + $healthUrl = ([System.Uri]::new([System.Uri]$env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT, 'health')).AbsoluteUri + $health = Invoke-RestMethod -Uri $healthUrl -TimeoutSec 5 -ErrorAction Stop + if ($health.connectivityProbes -lt 1 -or $health.gateway20Requests -lt 1) { + throw "Gateway 2.0 CI leg did not exercise the Gateway 2.0 path (probes=$($health.connectivityProbes), requests=$($health.gateway20Requests))." + } + Write-Host "Hosted Gateway 2.0 traffic verified (probes=$($health.connectivityProbes), requests=$($health.gateway20Requests))." + } catch { + LogError "Failed to verify hosted Gateway 2.0 traffic: $($_.Exception.Message)" + } + } + + if ($env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) { + $hostProcess = Get-Process -Id ([int]$env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) -ErrorAction SilentlyContinue + } else { + $hostProcess = Get-Process azure_data_cosmos_emulator -ErrorAction SilentlyContinue + } + if ($hostProcess) { + $hostProcess | Stop-Process -Force -ErrorAction SilentlyContinue + $hostProcess | Wait-Process -Timeout 10 -ErrorAction SilentlyContinue + } +} + +elseif ($IsWindows) { $EmulatorPath = & "$PSScriptRoot\Get-CosmosEmulatorPath.ps1" if ($null -eq $EmulatorPath) { Write-Host "Unable to confirm Cosmos DB Emulator location, skipping shutdown." @@ -72,11 +97,16 @@ if ($IsWindows) { # Test-Setup.ps1. When the pipeline deploys a live Cosmos account, the connection # string is injected as a pipeline variable and must survive across packages. Write-Host "Clearing emulator environment variables." -if ($env:AZURE_COSMOS_CONNECTION_STRING -eq "emulator") { +if ($env:AZURE_COSMOS_CONNECTION_STRING -eq "emulator" -or + $env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $env:AZURE_COSMOS_CONNECTION_STRING = $null } $env:AZURE_COSMOS_TEST_MODE = $null $env:AZURE_COSMOS_EMULATOR_HOST = $null +$env:AZURE_COSMOS_INMEMORY_EMULATOR_PID = $null +$env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $null +$env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT = $null +$env:AZURE_COSMOS_INMEMORY_ACCOUNT_ENDPOINT = $null # Remove any --cfg=test_category="..." flag added by Test-Setup.ps1 or COSMOS_RUSTFLAGS. # The next package's setup will re-add the correct flag from COSMOS_RUSTFLAGS # (or from AZURE_COSMOS_EMULATOR_FLAVOR=vnext when running the vnext stage). diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index c4cd47d450e..3fbb76d560d 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -14,6 +14,131 @@ if ($env:COSMOS_RUSTFLAGS) { Write-Host "RUSTFLAGS appended with COSMOS_RUSTFLAGS: $env:RUSTFLAGS" } +# Hosted in-memory emulator path. The additional CI matrix sets one of the two +# flavors below so the existing emulator suites run against both Gateway V1 +# and Gateway 2.0 over cleartext HTTP/2. +if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { + $repoRoot = (Resolve-Path ([System.IO.Path]::Combine($PSScriptRoot, '..', '..', '..', '..'))).Path + $configuration = if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'inmemory-v2') { + [System.IO.Path]::Combine($repoRoot, 'sdk', 'cosmos', 'azure_data_cosmos_emulator', 'config', 'ci-gateway-v2.json') + } else { + [System.IO.Path]::Combine($repoRoot, 'sdk', 'cosmos', 'azure_data_cosmos_emulator', 'config', 'ci-gateway-v1.json') + } + $ready = $false + $expectedGateway20 = $env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'inmemory-v2' + $managementEndpoint = $env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT + $accountEndpoint = $env:AZURE_COSMOS_INMEMORY_ACCOUNT_ENDPOINT + if ($managementEndpoint -and $accountEndpoint) { + $healthUrl = ([System.Uri]::new([System.Uri]$managementEndpoint, 'health')).AbsoluteUri + try { + $response = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -ErrorAction Stop + $health = $response.Content | ConvertFrom-Json + $ready = $response.StatusCode -eq 200 -and $health.gateway20Enabled -eq $expectedGateway20 + } catch { + $ready = $false + } + } + + if (-not $ready) { + Get-Process azure_data_cosmos_emulator -ErrorAction SilentlyContinue | Stop-Process -Force + } + + if (-not $ready) { + LogGroupStart "Building hosted Cosmos DB in-memory emulator" + Push-Location $repoRoot + try { + Invoke-LoggedCommand 'cargo build -p azure_data_cosmos_emulator' + } finally { + Pop-Location + } + LogGroupEnd + + $executableName = if ($IsWindows) { + 'azure_data_cosmos_emulator.exe' + } else { + 'azure_data_cosmos_emulator' + } + $executable = [System.IO.Path]::Combine($repoRoot, 'target', 'debug', $executableName) + $stdout = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), 'azure-data-cosmos-emulator.out.log') + $stderr = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), 'azure-data-cosmos-emulator.err.log') + Remove-Item $stdout, $stderr -Force -ErrorAction SilentlyContinue + + LogGroupStart "Starting hosted Cosmos DB in-memory emulator" + $process = Start-Process ` + -FilePath $executable ` + -ArgumentList @('--config', $configuration) ` + -RedirectStandardOutput $stdout ` + -RedirectStandardError $stderr ` + -PassThru + $env:AZURE_COSMOS_INMEMORY_EMULATOR_PID = $process.Id.ToString() + $env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $expectedGateway20.ToString().ToLowerInvariant() + Write-Host "Started hosted emulator process $($process.Id) using '$configuration'." + + $deadline = (Get-Date).AddSeconds(60) + $readyRecord = $null + while ((Get-Date) -lt $deadline) { + if ($process.HasExited) { + break + } + if (-not $readyRecord -and (Test-Path $stdout)) { + $readyLine = Get-Content $stdout -ErrorAction SilentlyContinue | Select-Object -Last 1 + if ($readyLine) { + try { + $candidate = $readyLine | ConvertFrom-Json -ErrorAction Stop + if ($candidate.event -eq 'ready') { + $readyRecord = $candidate + $managementEndpoint = [string]$readyRecord.managementEndpoint + $accountEndpoint = [string]$readyRecord.accountEndpoint + $hasGateway20 = @($readyRecord.regions | Where-Object { $_.gateway20Endpoint }).Count -gt 0 + if (-not $managementEndpoint -or -not $accountEndpoint -or $hasGateway20 -ne $expectedGateway20) { + throw 'Hosted emulator ready record does not match the requested gateway mode.' + } + $healthUrl = ([System.Uri]::new([System.Uri]$managementEndpoint, 'health')).AbsoluteUri + } + } catch { + $readyRecord = $null + Write-Host "Waiting for a valid hosted emulator ready record: $($_.Exception.Message)" + } + } + } + if ($readyRecord) { + try { + $response = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -ErrorAction Stop + $health = $response.Content | ConvertFrom-Json + if ($response.StatusCode -eq 200 -and $health.gateway20Enabled -eq $expectedGateway20) { + $ready = $true + break + } + } catch { + Write-Host "Waiting for hosted in-memory emulator readiness: $($_.Exception.Message)" + } + } + Start-Sleep -Seconds 1 + } + if (-not $ready) { + Get-Content $stdout, $stderr -ErrorAction SilentlyContinue | Write-Host + if (-not $process.HasExited) { + $process | Stop-Process -Force -ErrorAction SilentlyContinue + } + throw 'Hosted Cosmos DB in-memory emulator did not become ready within 60 seconds.' + } + LogGroupEnd + } else { + $env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $expectedGateway20.ToString().ToLowerInvariant() + } + + $env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT = $managementEndpoint + $env:AZURE_COSMOS_INMEMORY_ACCOUNT_ENDPOINT = $accountEndpoint + $emulatorKey = 'C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==' + $env:AZURE_COSMOS_CONNECTION_STRING = "AccountEndpoint=$accountEndpoint;AccountKey=$emulatorKey;" + $env:AZURE_COSMOS_TEST_MODE = 'required' + $env:RUSTFLAGS = $env:RUSTFLAGS -replace '\s*--cfg=test_category="[^"]*"', '' + $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator_inmemory`"" + $env:RUST_TEST_THREADS = '1' + Write-Host "Hosted emulator is ready; RUSTFLAGS set to: $env:RUSTFLAGS" + return +} + # Vnext (Linux) emulator path. Triggered by AZURE_COSMOS_EMULATOR_FLAVOR=vnext # (set as a matrix variable on the cosmos vnext leg in # sdk/cosmos/vnext-emulator-matrix.json). Manages the Docker container diff --git a/sdk/cosmos/inmemory-emulator-matrix.json b/sdk/cosmos/inmemory-emulator-matrix.json new file mode 100644 index 00000000000..bde1ad622df --- /dev/null +++ b/sdk/cosmos/inmemory-emulator-matrix.json @@ -0,0 +1,17 @@ +{ + "displayNames": { + "inmemory-v1": "inmemory_emulator_gateway_v1_dynamic_ports", + "inmemory-v2": "inmemory_emulator_gateway_2_0_dynamic_ports" + }, + "matrix": { + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "RustToolchainName": ["stable"], + "AZURE_COSMOS_EMULATOR_FLAVOR": ["inmemory-v1", "inmemory-v2"], + "ContinueOnError": ["true"] + } +} From 477b15f020b09a9735f6fd8a58a5e97a7d6f4ff8 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 01:09:11 +0000 Subject: [PATCH 31/44] Addressing code review comments --- Cargo.lock | 812 ++++++++++++------ eng/dict/crates.txt | 1 + .../templates/stages/archetype-sdk-client.yml | 4 + sdk/cosmos/azure_data_cosmos/build.rs | 2 +- .../docs/in-memory-emulator-spec.md | 9 +- sdk/cosmos/azure_data_cosmos_driver/build.rs | 2 +- .../src/in_memory_emulator/config.rs | 93 +- .../src/in_memory_emulator/gateway_v2.rs | 427 +++++++-- .../src/in_memory_emulator/operations.rs | 4 +- .../src/in_memory_emulator/store.rs | 150 ++-- .../in_memory_emulator/system_properties.rs | 4 +- .../src/models/effective_partition_key.rs | 15 +- .../emulator_tests/driver_fault_injection.rs | 46 +- .../in_memory_emulator_tests/error_cases.rs | 4 +- .../tests/in_memory_emulator_tests/mod.rs | 3 +- .../point_operations.rs | 63 ++ .../in_memory_emulator_tests/split_merge.rs | 182 +++- .../azure_data_cosmos_emulator/Cargo.toml | 1 + ...1_build_memory_backed_sdk_test_emulator.md | 10 +- .../azure_data_cosmos_emulator/docs/plan.md | 13 +- .../azure_data_cosmos_emulator/src/config.rs | 274 +++++- .../src/data_plane.rs | 23 +- .../src/gateway_v2.rs | 70 +- .../azure_data_cosmos_emulator/src/main.rs | 36 +- .../src/management.rs | 601 +++++++++---- sdk/cosmos/ci.yml | 15 +- .../eng/scripts/Invoke-CosmosTestCleanup.ps1 | 14 - .../eng/scripts/Invoke-CosmosTestSetup.ps1 | 9 +- sdk/cosmos/inmemory-emulator-matrix.json | 3 +- sdk/cosmos/vnext-emulator-matrix.json | 3 +- 30 files changed, 2121 insertions(+), 772 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e615593c344..a70d87b9e5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,18 +110,18 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] [[package]] name = "arrayvec" -version = "0.7.8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-compression" @@ -165,7 +165,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -176,7 +176,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -187,15 +187,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -203,15 +203,14 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", "dunce", "fs_extra", - "pkg-config", ] [[package]] @@ -322,7 +321,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -461,7 +460,7 @@ checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "tracing", ] @@ -471,7 +470,7 @@ version = "1.1.0-beta.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "tokio", "tracing", "tracing-subscriber", @@ -518,7 +517,7 @@ dependencies = [ "flate2", "futures", "include-file", - "rand 0.10.2", + "rand 0.10.1", "rand_chacha 0.10.0", "reqwest", "serde", @@ -538,7 +537,7 @@ dependencies = [ "azure_core_test", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "tokio", ] @@ -598,7 +597,7 @@ dependencies = [ "h2", "percent-encoding", "quick-xml", - "rand 0.10.2", + "rand 0.10.1", "reqwest", "serde", "serde_json", @@ -646,7 +645,7 @@ version = "0.3.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "tracing", ] @@ -664,7 +663,7 @@ dependencies = [ "futures", "hdrhistogram", "hostname", - "rand 0.10.2", + "rand 0.10.1", "serde", "serde_json", "sysinfo", @@ -735,7 +734,7 @@ dependencies = [ "futures", "hmac", "percent-encoding", - "rand 0.10.2", + "rand 0.10.1", "rand_chacha 0.10.0", "rustc_version", "sha2", @@ -780,7 +779,7 @@ dependencies = [ "azure_core_test", "azure_identity 1.0.0", "futures", - "rand 0.10.2", + "rand 0.10.1", "rand_chacha 0.10.0", "serde", "serde_json", @@ -804,7 +803,7 @@ dependencies = [ "futures", "include-file", "openssl", - "rand 0.10.2", + "rand 0.10.1", "rustc_version", "serde", "serde_json", @@ -826,7 +825,7 @@ dependencies = [ "criterion", "futures", "include-file", - "rand 0.10.2", + "rand 0.10.1", "rand_chacha 0.10.0", "rustc_version", "serde", @@ -849,7 +848,7 @@ dependencies = [ "clap", "futures", "include-file", - "rand 0.10.2", + "rand 0.10.1", "rustc_version", "serde", "serde_json", @@ -883,7 +882,7 @@ dependencies = [ "futures", "percent-encoding", "pin-project", - "rand 0.10.2", + "rand 0.10.1", "serde", "serde_json", "time", @@ -927,7 +926,7 @@ dependencies = [ "azure_storage_common", "azure_storage_sas", "futures", - "rand 0.10.2", + "rand 0.10.1", "serde", "time", "tokio", @@ -982,15 +981,15 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitvec" -version = "1.1.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium", @@ -1009,9 +1008,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.7.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" dependencies = [ "borsh-derive", "bytes", @@ -1020,22 +1019,22 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.7.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] name = "bumpalo" -version = "3.20.3" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytecheck" @@ -1067,9 +1066,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cast" @@ -1091,16 +1090,16 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.119", + "syn 2.0.117", "tempfile", "toml", ] [[package]] name = "cc" -version = "1.2.67" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1122,9 +1121,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -1133,9 +1132,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.45" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "num-traits", ] @@ -1169,9 +1168,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.2" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1179,9 +1178,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1198,7 +1197,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -1387,18 +1386,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1415,9 +1414,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1456,7 +1455,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -1467,14 +1466,14 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] name = "der" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" dependencies = [ "pem-rfc7468", "zeroize", @@ -1486,6 +1485,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ + "powerfmt", "serde_core", ] @@ -1502,13 +1502,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -1531,9 +1531,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.16.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "equivalent" @@ -1685,6 +1685,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1777,7 +1783,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -1848,16 +1854,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasm-bindgen", + "wasip2", + "wasip3", ] [[package]] @@ -1878,9 +1884,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1915,6 +1921,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1963,9 +1978,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", "itoa", @@ -1973,9 +1988,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", @@ -1983,9 +1998,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", @@ -2008,15 +2023,15 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.10.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -2095,7 +2110,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.5", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2183,6 +2198,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2218,7 +2239,7 @@ checksum = "babf2fc9613fcd8bf298719ad975819fc6ab212d3a60de4d2eb55554c7bfd03d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -2306,7 +2327,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -2325,27 +2346,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] name = "jobserver" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", + "once_cell", "wasm-bindgen", ] @@ -2377,6 +2399,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" version = "0.2.186" @@ -2406,9 +2434,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" @@ -2439,9 +2467,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.3" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "mime" @@ -2467,9 +2495,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -2565,9 +2593,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.81" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags", "cfg-if", @@ -2585,7 +2613,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -2596,9 +2624,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.117" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -2668,7 +2696,7 @@ dependencies = [ "futures-util", "opentelemetry 0.31.0", "percent-encoding", - "rand 0.9.5", + "rand 0.9.4", "thiserror", ] @@ -2684,7 +2712,7 @@ dependencies = [ "opentelemetry 0.32.0", "percent-encoding", "portable-atomic", - "rand 0.9.5", + "rand 0.9.4", "thiserror", "tokio", ] @@ -2696,7 +2724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.6", "serde", ] @@ -2771,7 +2799,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -2850,6 +2878,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -2888,7 +2926,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -2932,9 +2970,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -2943,7 +2981,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.5", + "socket2 0.6.3", "thiserror", "tokio", "tracing", @@ -2952,16 +2990,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.4.3", + "getrandom 0.3.4", "lru-slab", - "rand 0.10.2", - "rand_pcg", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -2975,23 +3012,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.5", + "socket2 0.6.3", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3016,9 +3053,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3028,9 +3065,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3038,12 +3075,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", - "getrandom 0.4.3", + "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -3102,15 +3139,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] - [[package]] name = "rayon" version = "1.12.0" @@ -3142,9 +3170,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3154,9 +3182,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3165,9 +3193,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.11" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "rend" @@ -3180,9 +3208,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64 0.22.1", "bytes", @@ -3265,15 +3293,15 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.42.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" dependencies = [ "arrayvec", "borsh", "bytes", "num-traits", - "rand 0.8.7", + "rand 0.8.6", "rkyv", "serde", "serde_json", @@ -3282,15 +3310,15 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.28" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -3316,9 +3344,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "once_cell", @@ -3330,9 +3358,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3342,9 +3370,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -3391,9 +3419,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3410,6 +3438,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3425,6 +3462,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "seahash" version = "4.1.0" @@ -3472,9 +3515,9 @@ dependencies = [ [[package]] name = "serde_amqp" -version = "0.14.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc615f24778bb6d92510fe82afc74f99fb03e6ddbfd75927356fe681c1b6037" +checksum = "e76738e7a058df01b5b33194359930a1aa5bc233dc07e510f458aca47189aea2" dependencies = [ "bytes", "indexmap 2.14.0", @@ -3496,7 +3539,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -3526,14 +3569,14 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", @@ -3561,7 +3604,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -3596,27 +3639,28 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.5.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ "futures-executor", "futures-util", "log", "once_cell", "parking_lot", + "scc", "serial_test_derive", ] [[package]] name = "serial_test_derive" -version = "3.5.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -3641,9 +3685,9 @@ dependencies = [ [[package]] name = "shlex" -version = "2.0.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" @@ -3657,15 +3701,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.10" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simd_cesu8" -version = "1.2.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" dependencies = [ "rustc_version", "simdutf8", @@ -3685,9 +3729,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" @@ -3701,9 +3745,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3740,9 +3784,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.119" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -3766,7 +3810,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -3806,7 +3850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3829,25 +3873,26 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] name = "thread_local" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.53" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde_core", @@ -3857,15 +3902,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -3893,9 +3938,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -3917,7 +3962,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.5", + "socket2 0.6.3", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -3931,7 +3976,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4025,14 +4070,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.4", + "winnow 1.0.3", ] [[package]] @@ -4041,14 +4086,14 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.4", + "winnow 1.0.3", ] [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tonic" @@ -4091,7 +4136,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.7", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -4171,7 +4216,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4221,7 +4266,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4238,9 +4283,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" [[package]] name = "typenum" -version = "1.20.1" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "typespec" @@ -4283,7 +4328,7 @@ dependencies = [ "dyn-clone", "futures", "pin-project", - "rand 0.10.2", + "rand 0.10.1", "reqwest", "rust_decimal", "serde", @@ -4308,7 +4353,7 @@ dependencies = [ "futures", "gloo-timers", "pin-project", - "rand 0.10.2", + "rand 0.10.1", "reqwest", "rust_decimal", "serde", @@ -4332,7 +4377,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4344,7 +4389,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "syn 2.0.119", + "syn 2.0.117", "tokio", "typespec_client_core 1.1.0", ] @@ -4357,9 +4402,15 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.3" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -4429,24 +4480,24 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.4.2", "js-sys", - "rand 0.10.2", + "rand 0.10.1", "uuid-rng-internal", "wasm-bindgen", ] [[package]] name = "uuid-rng-internal" -version = "1.24.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dff7387287837acf196f2596a6216f94c555d947d5331e1cc48393131020119" +checksum = "1409d2323564d9b8d01192a77a0604a46bf29eb42e9b535aca34dec65482652b" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.4.2", ] [[package]] @@ -4494,18 +4545,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.4+wasi-0.2.12" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -4517,9 +4577,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -4527,9 +4587,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4537,26 +4597,48 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.14.0", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.5.0" @@ -4570,6 +4652,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "semver", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -4586,9 +4680,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -4606,9 +4700,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ "rustls-pki-types", ] @@ -4651,7 +4745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ "windows-core 0.57.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4684,7 +4778,7 @@ dependencies = [ "windows-implement 0.57.0", "windows-interface 0.57.0", "windows-result 0.1.2", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4719,7 +4813,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4730,7 +4824,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4741,7 +4835,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4752,7 +4846,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4777,7 +4871,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4804,7 +4898,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -4822,14 +4925,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -4847,48 +4967,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.15" @@ -4897,19 +5065,107 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] name = "winnow" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.14.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.14.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.3" @@ -4927,9 +5183,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4944,28 +5200,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -4985,15 +5241,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" @@ -5025,7 +5281,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 2.0.117", ] [[package]] @@ -5044,15 +5300,15 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" -version = "1.0.23" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zopfli" diff --git a/eng/dict/crates.txt b/eng/dict/crates.txt index 1b1c49b14ed..3d31ec5ba3e 100644 --- a/eng/dict/crates.txt +++ b/eng/dict/crates.txt @@ -20,6 +20,7 @@ azure_data_cosmos azure_data_cosmos_benchmarks azure_data_cosmos_driver azure_data_cosmos_driver_native +azure_data_cosmos_emulator azure_data_cosmos_macros azure_data_cosmos_perf azure_identity diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 8167c3d2cbf..28672aa05dd 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -55,6 +55,9 @@ parameters: - name: AdditionalMatrixConfigs type: object default: [] +- name: BuildOnlyAdditionalMatrixConfigs + type: object + default: [] - name: MatrixFilters type: object default: [] @@ -122,6 +125,7 @@ extends: MatrixConfigs: - ${{ parameters.MatrixConfigs }} - ${{ parameters.AdditionalMatrixConfigs }} + - ${{ parameters.BuildOnlyAdditionalMatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} diff --git a/sdk/cosmos/azure_data_cosmos/build.rs b/sdk/cosmos/azure_data_cosmos/build.rs index 758eb1dec9d..1a5b3d72822 100644 --- a/sdk/cosmos/azure_data_cosmos/build.rs +++ b/sdk/cosmos/azure_data_cosmos/build.rs @@ -7,6 +7,6 @@ fn main() { // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. println!( - "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"multi_write\", \"split\", \"gateway_v2\", \"gateway_v2_multi_region\"))" + "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"emulator_inmemory_gateway_v2\", \"multi_write\", \"split\", \"gateway_v2\", \"gateway_v2_multi_region\"))" ); } diff --git a/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md b/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md index c6288a44b41..e1395dfafb9 100644 --- a/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md +++ b/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md @@ -59,7 +59,9 @@ An **in-memory emulator** that intercepts requests at the `HttpClient` transport ### Non-Goals (This Phase) - Bulk / Patch operations (return hard-coded errors). -- Gateway 2.0 transport mode (skip for now — will come later). +- Network hosting remains outside the in-process `HttpClient` interception contract described by + this document. The separate `azure_data_cosmos_emulator` host supports Gateway V1 and a scoped + Gateway 2.0 adapter; see `sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md`. - Change feed. - Stored procedures / triggers / UDFs. - Complete Cosmos SQL service parity beyond the local query evaluator and local query-plan analyzer. @@ -1124,7 +1126,10 @@ The `GET /dbs/{db}/colls/{coll}/pkranges` feed reflects the updated topology: ## 18. Unsupported Operations -Bulk, Patch, ChangeFeed, stored procedures, triggers, UDFs, and Gateway 2.0 transport mode remain unsupported and return **501 Not Implemented** or the closest service-shaped error for that route. +Bulk, Patch, ChangeFeed, stored procedures, triggers, and UDFs remain unsupported and return +**501 Not Implemented** or the closest service-shaped error for that route. The in-process client +does not select a network transport, while the separate hosted emulator supports the documented +subset of Gateway 2.0 over HTTP/2. Query support is intentionally scoped to the local SQL evaluator and local query-plan analyzer used by the SDK tests. Transactional batch supports document operations within one logical partition and rolls back the whole batch on failure. diff --git a/sdk/cosmos/azure_data_cosmos_driver/build.rs b/sdk/cosmos/azure_data_cosmos_driver/build.rs index 5342ab0f052..7538a3e771e 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/build.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/build.rs @@ -7,6 +7,6 @@ fn main() { // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. println!( - "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"multi_write\", \"multi_region\", \"gateway_v2\", \"gateway_v2_multi_region\", \"native_query_plan\"))" + "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"emulator_inmemory\", \"emulator_inmemory_gateway_v2\", \"multi_write\", \"multi_region\", \"gateway_v2\", \"gateway_v2_multi_region\", \"native_query_plan\"))" ); } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs index 8bfa9d68a40..11255267d3a 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/config.rs @@ -4,7 +4,7 @@ // cspell:ignore PRNG //! Virtual account configuration for the in-memory emulator. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -52,14 +52,23 @@ impl VirtualAccountConfig { .with_message("at least one region is required") .build()); } - // Auto-assign monotonically increasing region IDs by position for any - // region that did not have one set explicitly via `with_region_id`. - // Using `0` as the sentinel means callers that explicitly pass - // `with_region_id(0)` to the *first* region get the same effective ID - // they would have been auto-assigned anyway. + // Auto-assign monotonically increasing region IDs by position only + // when the caller did not set one explicitly. for (idx, r) in regions.iter_mut().enumerate() { - if r.region_id == 0 { - r.region_id = idx as u64; + r.region_id.get_or_insert(idx as u64); + } + let mut region_ids = HashSet::with_capacity(regions.len()); + for region in ®ions { + let region_id = region.region_id.expect("region IDs were assigned above"); + if !region_ids.insert(region_id) { + return Err(crate::error::CosmosError::builder() + .with_status(crate::error::CosmosStatus::new( + azure_core::http::StatusCode::BadRequest, + )) + .with_message(format!( + "region ID {region_id} is configured more than once" + )) + .build()); } } Ok(Self { @@ -267,7 +276,7 @@ impl VirtualAccountConfig { return Some(&r.name); } #[cfg(feature = "__internal_in_memory_emulator")] - if r.thin_client_url.as_ref().is_some_and(matches) { + if r.gateway_v2_url.as_ref().is_some_and(matches) { return Some(&r.name); } } @@ -279,7 +288,7 @@ impl VirtualAccountConfig { self.regions .iter() .find(|r| r.name == region_name) - .map(|r| r.region_id) + .and_then(|r| r.region_id) .unwrap_or(0) } } @@ -290,8 +299,8 @@ pub struct VirtualRegion { name: String, gateway_url: Url, #[cfg(feature = "__internal_in_memory_emulator")] - thin_client_url: Option, - region_id: u64, + gateway_v2_url: Option, + region_id: Option, } impl VirtualRegion { @@ -304,22 +313,22 @@ impl VirtualRegion { name: name.to_string(), gateway_url, #[cfg(feature = "__internal_in_memory_emulator")] - thin_client_url: None, - region_id: 0, + gateway_v2_url: None, + region_id: None, } } /// Configures the Gateway V2 thin-client endpoint for this region. #[cfg(feature = "__internal_in_memory_emulator")] #[doc(hidden)] - pub fn with_thin_client_url(mut self, url: Url) -> Self { - self.thin_client_url = Some(url); + pub fn with_gateway_v2_url(mut self, url: Url) -> Self { + self.gateway_v2_url = Some(url); self } /// Creates a new region with an explicit region ID. pub fn with_region_id(mut self, id: u64) -> Self { - self.region_id = id; + self.region_id = Some(id); self } @@ -334,12 +343,13 @@ impl VirtualRegion { /// Returns the Gateway V2 thin-client endpoint when hosted externally. #[cfg(feature = "__internal_in_memory_emulator")] #[doc(hidden)] - pub(crate) fn thin_client_url(&self) -> Option<&Url> { - self.thin_client_url.as_ref() + pub(crate) fn gateway_v2_url(&self) -> Option<&Url> { + self.gateway_v2_url.as_ref() } pub fn region_id(&self) -> u64 { self.region_id + .expect("VirtualAccountConfig assigns every region an ID") } } @@ -678,3 +688,48 @@ impl Default for ContainerConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn region(name: &str) -> VirtualRegion { + VirtualRegion::new( + name, + Url::parse(&format!( + "https://{}.emulator.local", + name.to_ascii_lowercase() + )) + .unwrap(), + ) + } + + #[test] + fn assigns_region_ids_by_position_when_omitted() { + let config = VirtualAccountConfig::new(vec![region("East"), region("West")]).unwrap(); + assert_eq!(config.regions()[0].region_id(), 0); + assert_eq!(config.regions()[1].region_id(), 1); + } + + #[test] + fn preserves_explicit_zero_for_non_first_region() { + let config = VirtualAccountConfig::new(vec![ + region("East").with_region_id(1), + region("West").with_region_id(0), + ]) + .unwrap(); + assert_eq!(config.regions()[0].region_id(), 1); + assert_eq!(config.regions()[1].region_id(), 0); + } + + #[test] + fn rejects_duplicate_effective_region_ids() { + let error = + VirtualAccountConfig::new(vec![region("East").with_region_id(1), region("West")]) + .unwrap_err(); + assert_eq!( + error.status().status_code(), + azure_core::http::StatusCode::BadRequest + ); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs index 76b1c0500bc..18c2866b04f 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs @@ -12,6 +12,7 @@ use azure_core::{ }; use uuid::Uuid; +use crate::models::effective_partition_key::prefix_range_end_hex; use crate::{ driver::transport::rntbd::{ tokens::{RntbdRequestToken, TokenValue}, @@ -33,14 +34,39 @@ impl InMemoryEmulatorHttpClient { let frame = RntbdRequestFrame::read(request_body.as_ref()).map_err(gateway_v2_bad_request)?; let activity_id = frame.activity_id; - let request = decode_request(request, frame, self.store().config().consistency())?; + let request = match decode_request(request, frame, self.store().config().consistency()) { + Ok(request) => request, + Err(error) => return encode_error_response(error, activity_id).await, + }; let response = self.execute_request(&request).await?; encode_response(response, activity_id).await } } +async fn encode_error_response( + error: crate::error::CosmosError, + activity_id: Uuid, +) -> crate::error::Result { + let mut headers = Headers::new(); + headers.insert("x-ms-activity-id", activity_id.to_string()); + if let Some(sub_status) = error.status().sub_status() { + headers.insert("x-ms-substatus", sub_status.value().to_string()); + } + let body = serde_json::to_vec(&serde_json::json!({ + "code": "BadRequest", + "message": error.to_string(), + })) + .map_err(gateway_v2_internal_error)?; + encode_response( + AsyncRawResponse::from_bytes(error.status().status_code(), headers, body), + activity_id, + ) + .await +} + #[derive(Default)] struct RequestMetadata { + payload_present: Option, database: Option, collection: Option, document: Option, @@ -72,7 +98,16 @@ fn decode_request( frame.resource_type ))); } - let mut metadata = decode_metadata(frame.metadata); + let body_present = frame.body.is_some(); + let mut metadata = decode_metadata(frame.metadata)?; + let payload_present = metadata + .payload_present + .ok_or_else(|| gateway_v2_bad_request("RNTBD request is missing PayloadPresent"))?; + if payload_present != body_present { + return Err(gateway_v2_bad_request(format!( + "RNTBD PayloadPresent was {payload_present} but body presence was {body_present}" + ))); + } if metadata.allow_tentative_writes { return Err(gateway_v2_bad_request( "hosted Gateway V2 does not yet support AllowTentativeWrites", @@ -102,7 +137,15 @@ fn decode_request( { if let Some(effective_partition_key) = metadata.effective_partition_key.as_ref() { metadata.start_epk = Some(effective_partition_key.clone()); - metadata.end_epk = Some(format!("{effective_partition_key}FF")); + let bytes = effective_partition_key + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let value = std::str::from_utf8(pair).map_err(gateway_v2_bad_request)?; + u8::from_str_radix(value, 16).map_err(gateway_v2_bad_request) + }) + .collect::>>()?; + metadata.end_epk = Some(prefix_range_end_hex(&bytes)); } } if matches!( @@ -148,6 +191,7 @@ fn decode_request( } }; + let outer_path = outer_request.url().path().to_owned(); let mut url = outer_request.url().clone(); url.set_query(None); url.set_fragment(None); @@ -169,6 +213,12 @@ fn decode_request( segments.push(document); } } + if outer_path != url.path() { + return Err(gateway_v2_bad_request(format!( + "Gateway 2.0 outer path '{outer_path}' does not match RNTBD target '{}'", + url.path() + ))); + } let mut request = Request::new(url, method); request.headers_mut().insert( @@ -253,90 +303,146 @@ fn decode_request( Ok(request) } -fn decode_metadata(tokens: Vec) -> RequestMetadata { +fn decode_metadata( + tokens: Vec, +) -> crate::error::Result { let mut metadata = RequestMetadata::default(); for token in tokens { let Ok(kind) = RntbdRequestToken::try_from(token.id.value()) else { continue; }; match kind { - RntbdRequestToken::DatabaseName => metadata.database = token_string(token.value), - RntbdRequestToken::CollectionName => metadata.collection = token_string(token.value), - RntbdRequestToken::DocumentName => metadata.document = token_string(token.value), - RntbdRequestToken::PartitionKey => metadata.partition_key = token_string(token.value), + RntbdRequestToken::DatabaseName => { + metadata.database = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::CollectionName => { + metadata.collection = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::DocumentName => { + metadata.document = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::PartitionKey => { + metadata.partition_key = Some(expect_string(kind, token.value)?) + } RntbdRequestToken::PartitionKeyRangeId => { - metadata.partition_key_range_id = token_string(token.value) + metadata.partition_key_range_id = Some(expect_string(kind, token.value)?) } RntbdRequestToken::ContinuationToken => { - metadata.continuation = token_string(token.value) + metadata.continuation = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::SessionToken => { + metadata.session_token = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::Match => { + metadata.match_condition = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::StartEpkHash => { + metadata.start_epk = Some(expect_hex(kind, token.value)?) + } + RntbdRequestToken::EndEpkHash => { + metadata.end_epk = Some(expect_hex(kind, token.value)?) } - RntbdRequestToken::SessionToken => metadata.session_token = token_string(token.value), - RntbdRequestToken::Match => metadata.match_condition = token_string(token.value), - RntbdRequestToken::StartEpkHash => metadata.start_epk = token_hex(token.value), - RntbdRequestToken::EndEpkHash => metadata.end_epk = token_hex(token.value), RntbdRequestToken::EffectivePartitionKey => { - metadata.effective_partition_key = token_hex(token.value) + metadata.effective_partition_key = Some(expect_hex(kind, token.value)?) } RntbdRequestToken::PageSize => { - if let TokenValue::ULong(value) = token.value { - metadata.page_size = Some(value); - } + metadata.page_size = Some(expect_ulong(kind, token.value)?); } RntbdRequestToken::ReturnPreference => { - metadata.return_minimal = - matches!(token.value, TokenValue::Byte(value) if value != 0) + metadata.return_minimal = expect_byte(kind, token.value)? != 0 } RntbdRequestToken::SupportedQueryFeatures => { - metadata.supported_query_features = token_string(token.value) + metadata.supported_query_features = Some(expect_string(kind, token.value)?) + } + RntbdRequestToken::QueryVersion => { + metadata.query_version = Some(expect_small_string(kind, token.value)?) } - RntbdRequestToken::QueryVersion => metadata.query_version = token_string(token.value), RntbdRequestToken::AllowTentativeWrites => { - metadata.allow_tentative_writes = - matches!(token.value, TokenValue::Byte(value) if value != 0) + metadata.allow_tentative_writes = expect_byte(kind, token.value)? != 0 } RntbdRequestToken::ConsistencyLevel => { - if let TokenValue::Byte(value) = token.value { - metadata.consistency_level = Some(value); - } + metadata.consistency_level = Some(expect_byte(kind, token.value)?); } RntbdRequestToken::ReadConsistencyStrategy => { - if let TokenValue::Byte(value) = token.value { - metadata.read_consistency_strategy = Some(value); + metadata.read_consistency_strategy = Some(expect_byte(kind, token.value)?); + } + RntbdRequestToken::PayloadPresent => { + if metadata.payload_present.is_some() { + return Err(gateway_v2_bad_request( + "RNTBD request contains duplicate PayloadPresent tokens", + )); } + metadata.payload_present = Some(expect_byte(kind, token.value)? != 0); } - RntbdRequestToken::ResourceId - | RntbdRequestToken::AuthorizationToken - | RntbdRequestToken::PayloadPresent - | RntbdRequestToken::Date + RntbdRequestToken::ResourceId => { + expect_bytes(kind, token.value)?; + } + RntbdRequestToken::AuthorizationToken | RntbdRequestToken::CollectionRid - | RntbdRequestToken::TransportRequestId - | RntbdRequestToken::SDKSupportedCapabilities | RntbdRequestToken::GlobalDatabaseAccountName => { - // The hosted adapter already has the corresponding routing or - // transport context; these tokens do not alter store semantics. + expect_string(kind, token.value)?; + } + RntbdRequestToken::Date => { + expect_small_string(kind, token.value)?; + } + RntbdRequestToken::TransportRequestId | RntbdRequestToken::SDKSupportedCapabilities => { + expect_ulong(kind, token.value)?; } } } - metadata + Ok(metadata) } -fn token_string(value: TokenValue) -> Option { +fn expect_string(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result { match value { - TokenValue::SmallString(value) - | TokenValue::String(value) - | TokenValue::ULongString(value) => Some(value), - _ => None, + TokenValue::String(value) => Ok(value), + other => Err(wrong_token_type(kind, "String", other)), } } -fn token_hex(value: TokenValue) -> Option { - let bytes = match value { - TokenValue::SmallBytes(value) - | TokenValue::Bytes(value) - | TokenValue::ULongBytes(value) => value, - _ => return None, - }; - Some(bytes.iter().map(|byte| format!("{byte:02X}")).collect()) +fn expect_small_string(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result { + match value { + TokenValue::SmallString(value) => Ok(value), + other => Err(wrong_token_type(kind, "SmallString", other)), + } +} + +fn expect_bytes(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result> { + match value { + TokenValue::Bytes(value) => Ok(value), + other => Err(wrong_token_type(kind, "Bytes", other)), + } +} + +fn expect_hex(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result { + Ok(expect_bytes(kind, value)? + .iter() + .map(|byte| format!("{byte:02X}")) + .collect()) +} + +fn expect_byte(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result { + match value { + TokenValue::Byte(value) => Ok(value), + other => Err(wrong_token_type(kind, "Byte", other)), + } +} + +fn expect_ulong(kind: RntbdRequestToken, value: TokenValue) -> crate::error::Result { + match value { + TokenValue::ULong(value) => Ok(value), + other => Err(wrong_token_type(kind, "ULong", other)), + } +} + +fn wrong_token_type( + kind: RntbdRequestToken, + expected: &str, + actual: TokenValue, +) -> crate::error::CosmosError { + gateway_v2_bad_request(format!( + "RNTBD token {kind:?} must use {expected}, got {actual:?}" + )) } fn consistency_wire_byte(value: ConsistencyLevel) -> u8 { @@ -390,14 +496,14 @@ async fn encode_response( "x-ms-item-lsn", "x-ms-global-committed-lsn", "x-ms-documentdb-query-metrics", - "x-ms-index-utilization", + "x-ms-cosmos-index-utilization", ] { if let Some(value) = header_string(headers, name) { outer_headers.insert(name, value); } } Ok(AsyncRawResponse::from_bytes( - StatusCode::Ok, + status.status_code(), outer_headers, body, )) @@ -449,7 +555,7 @@ mod tests { async fn create_item_round_trips_through_gateway_v2() { let thin_url = Url::parse("http://127.0.0.1:18444/").unwrap(); let region = VirtualRegion::new("East US", "http://127.0.0.1:18081/".parse().unwrap()) - .with_thin_client_url(thin_url.clone()); + .with_gateway_v2_url(thin_url.clone()); let emulator = InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![region]).unwrap()); let store = emulator.store(); @@ -488,7 +594,10 @@ mod tests { }; let mut bytes = Vec::new(); frame.write(&mut bytes).unwrap(); - let mut request = Request::new(thin_url, Method::Post); + let mut request = Request::new( + thin_url.join("dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); request.set_body(bytes); let response = emulator @@ -499,13 +608,126 @@ mod tests { .await .unwrap(); - assert_eq!(response.status(), StatusCode::Ok); + assert_eq!(response.status(), StatusCode::Created); let response = RntbdResponse::read(response.body().as_ref()).unwrap(); assert_eq!(response.status.status_code(), StatusCode::Created); assert_eq!(response.activity_id, activity_id); assert!(!response.body.is_empty()); } + #[tokio::test] + async fn service_not_found_uses_matching_outer_and_inner_status() { + let gateway_v2_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", "http://127.0.0.1:18081/".parse().unwrap()) + .with_gateway_v2_url(gateway_v2_url.clone()); + let emulator = + InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![region]).unwrap()); + emulator.store().create_database("db"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + emulator + .store() + .create_container("db", "coll", partition_key); + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Read, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::document_name("missing".to_owned()), + Token::partition_key(r#"["pk1"]"#.to_owned()), + Token::payload_present(false), + ], + body: None, + }; + let mut bytes = Vec::new(); + frame.write(&mut bytes).unwrap(); + let mut request = Request::new( + gateway_v2_url + .join("dbs/db/colls/coll/docs/missing") + .unwrap(), + Method::Post, + ); + request.set_body(bytes); + + let response = emulator + .execute_gateway_v2_request(&request) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NotFound); + let framed = RntbdResponse::read(response.body().as_ref()).unwrap(); + assert_eq!(framed.status.status_code(), StatusCode::NotFound); + } + + #[tokio::test] + async fn semantic_validation_error_is_framed_with_matching_status() { + let gateway_v2_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", "http://127.0.0.1:18081/".parse().unwrap()) + .with_gateway_v2_url(gateway_v2_url.clone()); + let emulator = + InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![region]).unwrap()); + let activity_id = Uuid::new_v4(); + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Create, + activity_id, + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::allow_tentative_writes(true), + Token::payload_present(true), + ], + body: Some(br#"{"id":"item"}"#.to_vec()), + }; + let mut bytes = Vec::new(); + frame.write(&mut bytes).unwrap(); + let mut request = Request::new( + gateway_v2_url.join("dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + request.set_body(bytes); + + let response = emulator + .execute_gateway_v2_request(&request) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BadRequest); + let framed = RntbdResponse::read(response.body().as_ref()).unwrap(); + assert_eq!(framed.status.status_code(), StatusCode::BadRequest); + assert_eq!(framed.activity_id, activity_id); + } + + #[tokio::test] + async fn throttled_response_uses_matching_outer_and_inner_status() { + let mut headers = Headers::new(); + headers.insert("x-ms-substatus", "3200"); + headers.insert("x-ms-retry-after-ms", "25"); + let response = encode_response( + AsyncRawResponse::from_bytes(StatusCode::TooManyRequests, headers, Vec::new()), + Uuid::new_v4(), + ) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::TooManyRequests); + let framed = RntbdResponse::read(response.body().as_ref()).unwrap(); + assert_eq!(framed.status.status_code(), StatusCode::TooManyRequests); + assert_eq!(framed.status.sub_status().unwrap().value(), 3200); + assert_eq!(framed.retry_after_ms, Some(25)); + } + #[test] fn effective_partition_key_scopes_query_range() { let frame = RntbdRequestFrame { @@ -516,10 +738,14 @@ mod tests { Token::database_name("db".to_owned()), Token::collection_name("coll".to_owned()), Token::effective_partition_key(vec![0x10, 0x20]), + Token::payload_present(true), ], body: Some(br#"{"query":"SELECT * FROM c"}"#.to_vec()), }; - let outer = Request::new(Url::parse("http://127.0.0.1:18444/").unwrap(), Method::Post); + let outer = Request::new( + Url::parse("http://127.0.0.1:18444/dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); let request = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap(); assert_eq!( @@ -546,12 +772,97 @@ mod tests { Token::database_name("db".to_owned()), Token::collection_name("coll".to_owned()), Token::allow_tentative_writes(true), + Token::payload_present(true), ], body: Some(br#"{"id":"item"}"#.to_vec()), }; - let outer = Request::new(Url::parse("http://127.0.0.1:18444/").unwrap(), Method::Post); + let outer = Request::new( + Url::parse("http://127.0.0.1:18444/dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); let error = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap_err(); assert!(error.to_string().contains("AllowTentativeWrites")); } + + #[test] + fn rejects_wrong_types_for_known_tokens() { + let outer = Request::new( + Url::parse("http://127.0.0.1:18444/dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + for token in [ + Token::new(RntbdRequestToken::Match, TokenValue::ULong(1)), + Token::new( + RntbdRequestToken::PageSize, + TokenValue::String("1".to_owned()), + ), + Token::new( + RntbdRequestToken::EffectivePartitionKey, + TokenValue::String("01".to_owned()), + ), + Token::new(RntbdRequestToken::StartEpkHash, TokenValue::Byte(1)), + Token::new(RntbdRequestToken::EndEpkHash, TokenValue::Byte(1)), + ] { + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Query, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::payload_present(true), + token, + ], + body: Some(br#"{"query":"SELECT * FROM c"}"#.to_vec()), + }; + let error = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap_err(); + assert!(error.to_string().contains("must use")); + } + } + + #[test] + fn rejects_payload_present_mismatches() { + let outer = Request::new( + Url::parse("http://127.0.0.1:18444/dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + for (payload_present, body) in [(true, None), (false, Some(Vec::new()))] { + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::ReadFeed, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::payload_present(payload_present), + ], + body, + }; + let error = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap_err(); + assert!(error.to_string().contains("PayloadPresent")); + } + } + + #[test] + fn rejects_outer_path_that_disagrees_with_frame_target() { + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::ReadFeed, + activity_id: Uuid::new_v4(), + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::payload_present(false), + ], + body: None, + }; + let outer = Request::new( + Url::parse("http://127.0.0.1:18444/wrong").unwrap(), + Method::Post, + ); + + let error = decode_request(&outer, frame, ConsistencyLevel::Session).unwrap_err(); + assert!(error.to_string().contains("does not match RNTBD target")); + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs index eaf3d65c264..09dd6730df4 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs @@ -4580,7 +4580,7 @@ fn handle_read( .with_request_charge(charge) .with_session_token(&token) .with_etag(&etag); - return Err(decorate_point_response(builder, headers, Some(lsn)).build()); + return Err(decorate_point_response(builder, headers, Some(item_lsn)).build()); } return Ok((body, etag, token, charge, lsn, item_lsn, headers)); } @@ -5069,7 +5069,7 @@ async fn handle_upsert_locked( if let Some(if_match) = parsed.if_match.as_ref() { if logical .get(&doc_id) - .is_none_or(|existing| *if_match != existing.etag) + .is_some_and(|existing| *if_match != existing.etag) { return Err(error_response( StatusCode::PreconditionFailed, diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs index fe4d02b05e3..bf7b8882bec 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/store.rs @@ -1132,90 +1132,21 @@ impl EmulatorStore { if let Some(state) = containers.get(&key) { if let Some(partition) = state.find_partition(&doc.epk) { if partition.is_locked() { - // Drop the read guards before scheduling the retry so - // the spawned task can re-acquire them. - drop(containers); - drop(regions); - self.defer_replication_during_lock( - target_region, - source_region, - db_id, - coll_id, - doc, - is_delete, - ); - return; + let mut deferred = partition.deferred_replications.write().unwrap(); + // The abort path serializes with this queue before it + // clears `locked`. Rechecking under the queue lock + // ensures a replication either joins the topology + // operation or applies to the unchanged parent. + if partition.is_locked() { + deferred.push((doc.clone(), is_delete)); + return; + } } apply_doc_to_partition(partition, doc, is_delete); } } } } - - /// Schedules a bounded retry of `apply_replication` while the EPK's - /// target partition is locked for split/merge. After the lock clears - /// `find_partition` returns the new child partition (split) or the - /// merged successor (merge), and the doc lands in the correct place. - /// Without this hop, late-arriving replicated writes during a split - /// land in the BTreeMap of the parent partition that is about to be - /// replaced — the doc snapshot taken inside `execute_split` misses it - /// and the document is silently lost. - fn defer_replication_during_lock( - self: &Arc, - target_region: &str, - source_region: &str, - db_id: &str, - coll_id: &str, - doc: &StoredDocument, - is_delete: bool, - ) { - const MAX_ATTEMPTS: u32 = 50; - const RETRY_DELAY: Duration = Duration::from_millis(20); - - let store = Arc::clone(self); - let target = target_region.to_string(); - let source = source_region.to_string(); - let db = db_id.to_string(); - let coll = coll_id.to_string(); - let document = doc.clone(); - let semaphore = Arc::clone(&self.replication_semaphore); - self.replication_tasks.lock().unwrap().spawn(async move { - let _permit = semaphore.acquire_owned().await.ok(); - for attempt in 0..MAX_ATTEMPTS { - tokio::time::sleep(RETRY_DELAY).await; - let still_locked = { - let regions = store.regions.read().unwrap(); - let Some(region_store) = regions.get(&target) else { - return; - }; - let containers = region_store.containers.read().unwrap(); - let key = (db.clone(), coll.clone()); - let Some(state) = containers.get(&key) else { - return; - }; - state - .find_partition(&document.epk) - .map(|p| p.is_locked()) - .unwrap_or(false) - }; - if !still_locked { - store.apply_replication(&target, &source, &db, &coll, &document, is_delete); - return; - } - if attempt + 1 == MAX_ATTEMPTS { - store.dropped_replications.fetch_add(1, Ordering::SeqCst); - tracing::warn!( - target_region = %target, - source_region = %source, - db_id = %db, - coll_id = %coll, - epk = %document.epk, - "in-memory emulator: dropping replicated write after extended split/merge lock", - ); - } - } - }); - } } impl std::fmt::Debug for EmulatorStore { @@ -1831,6 +1762,42 @@ impl ThroughputTracker { // --- Split / Merge --- impl EmulatorStore { + /// Validates that an explicit split EPK lies strictly inside an existing partition. + #[cfg(feature = "__internal_in_memory_emulator")] + #[doc(hidden)] + pub fn validate_split_epk( + &self, + db_id: &str, + coll_id: &str, + partition_id: u32, + split_epk: &Epk, + ) -> crate::error::Result<()> { + let region = self + .region(self.config.write_region_name()) + .ok_or_else(|| host_control_plane_error("write region does not exist"))?; + region + .with_container(db_id, coll_id, |state| { + let partition = state + .physical_partitions + .iter() + .find(|partition| partition.id == partition_id) + .ok_or_else(|| { + host_control_plane_error(format!( + "partition {partition_id} does not exist in {db_id}/{coll_id}" + )) + })?; + if *split_epk <= partition.epk_min || *split_epk >= partition.epk_max { + return Err(host_control_plane_error(format!( + "split EPK must lie strictly inside partition {partition_id}" + ))); + } + Ok(()) + }) + .ok_or_else(|| { + host_control_plane_error(format!("container {db_id}/{coll_id} does not exist")) + })? + } + /// Returns the EPK boundary a midpoint split would use for a physical partition. #[cfg(feature = "__internal_in_memory_emulator")] #[doc(hidden)] @@ -2268,20 +2235,7 @@ impl EmulatorStore { } = (match preview { Some(SplitPreview::Found { .. }) => preview.unwrap(), Some(SplitPreview::AbortUnlock) => { - // Outer `regions` read guard has been dropped here. - let regions = self.regions.read().unwrap(); - for region in regions.values() { - let containers = region.containers.read().unwrap(); - if let Some(state) = containers.get(&key) { - if let Some(p) = state - .physical_partitions - .iter() - .find(|p| p.id == partition_id) - { - p.locked.store(false, Ordering::SeqCst); - } - } - } + self.unlock_partitions(&key, &[partition_id]); return false; } None => return false, @@ -2570,7 +2524,15 @@ impl EmulatorStore { if let Some(state) = containers.get(key) { for partition in &state.physical_partitions { if partition_ids.contains(&partition.id) { - partition.locked.store(false, Ordering::SeqCst); + let deferred = { + let mut deferred = partition.deferred_replications.write().unwrap(); + let entries = std::mem::take(&mut *deferred); + partition.locked.store(false, Ordering::SeqCst); + entries + }; + for (doc, is_delete) in deferred { + apply_doc_to_partition(partition, &doc, is_delete); + } } } } @@ -2664,6 +2626,7 @@ impl EmulatorStore { return false; } }; + let new_container_etag = new_etag(); let regions = self.regions.read().unwrap(); for region in regions.values() { @@ -2825,6 +2788,7 @@ impl EmulatorStore { state.physical_partitions.remove(first_remove); state.physical_partitions.remove(second_remove); state.physical_partitions.push(child); + state.metadata.etag = new_container_etag.clone(); } } true diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs index 7d858c7f58c..a42fb427b6c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/system_properties.rs @@ -219,7 +219,7 @@ pub(crate) fn account_properties_to_json( .regions() .iter() .filter_map(|region| { - region.thin_client_url().map(|url| { + region.gateway_v2_url().map(|url| { serde_json::json!({ "name": region.name(), "databaseAccountEndpoint": url.as_str() @@ -232,7 +232,7 @@ pub(crate) fn account_properties_to_json( .iter() .filter(|region| config.is_write_region(region.name())) .filter_map(|region| { - region.thin_client_url().map(|url| { + region.gateway_v2_url().map(|url| { serde_json::json!({ "name": region.name(), "databaseAccountEndpoint": url.as_str() diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/effective_partition_key.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/effective_partition_key.rs index 1022f6ab795..4ee7c6c8ee7 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/effective_partition_key.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/effective_partition_key.rs @@ -16,6 +16,17 @@ use std::borrow::Cow; use std::fmt; use std::fmt::Write; +pub(crate) fn prefix_range_end_bytes(prefix: &[u8]) -> Vec { + let mut end = Vec::with_capacity(prefix.len() + 1); + end.extend_from_slice(prefix); + end.push(0xFF); + end +} + +pub(crate) fn prefix_range_end_hex(prefix: &[u8]) -> String { + bytes_to_hex_upper(&prefix_range_end_bytes(prefix)) +} + /// A newtype wrapping the raw effective partition key bytes. /// /// An `EffectivePartitionKey` is the result of hashing a [`PartitionKey`](crate::models::PartitionKey) @@ -219,9 +230,7 @@ impl EffectivePartitionKey { // `hash_v2_raw_bytes` masks byte 0 with 0x3F, so every EPK // component's first byte is in `[0x00, 0x3F]`. `0xFF` is greater // than any valid suffix byte. - let mut max_bytes = epk.as_bytes().to_vec(); - max_bytes.push(0xFF); - let max = Self::from_bytes(max_bytes); + let max = Self::from_bytes(prefix_range_end_bytes(epk.as_bytes())); Ok(epk..max) } else { Ok(epk.clone()..epk) diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs index 482edc40ffd..b9a133b3a64 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_fault_injection.rs @@ -371,27 +371,22 @@ pub async fn fault_injection_connection_error() -> Result<(), Box> { // // Each rule is scoped to `TransportKind::GatewayV2` via // `with_transport_kind(...)` so it only fires on Gateway 2.0 traffic. The -// emulator does not expose Gateway 2.0 endpoints, so these tests are gated -// behind the `gateway_v2` test category and rely on the -// `Session SingleRegion GatewayV2` CI matrix entry pointing at a -// pre-provisioned Gateway 2.0 account (see `sdk/cosmos/ci.yml` and the -// `AZURE_COSMOS_GW_V2_ENDPOINT` / `AZURE_COSMOS_GW_V2_KEY` plumbing in the -// driver test framework's `resolve_test_env`). +// Single-region protocol behaviors also run against the hosted Gateway 2.0 +// emulator. Tests whose names promise regional movement remain on the live +// multi-region account until the hosted CI topology includes multiple regions. /// Gateway 2.0 503 Service Unavailable should trigger regional failover. /// /// The rule is scoped to [`TransportKind::GatewayV2`] so it does not also /// fire on standard-gateway requests issued during account discovery. The -/// emulator does not yet expose Gateway 2.0 endpoints, so this test is -/// gated behind the `gateway_v2` test category until CI gains a Gateway 2.0 -/// account. +/// This regional-failover test still requires a multi-region Gateway 2.0 account. #[tokio::test] #[cfg_attr( not(any( test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires a live multi-region Gateway 2.0 account" )] pub async fn gateway_v2_service_unavailable_triggers_regional_failover( ) -> Result<(), Box> { @@ -445,15 +440,15 @@ pub async fn gateway_v2_service_unavailable_triggers_regional_failover( /// across regions without risking duplicates). /// /// The rule is scoped to [`TransportKind::GatewayV2`] so it does not affect -/// standard-gateway traffic. The emulator does not yet expose Gateway 2.0 -/// endpoints, so this test is gated behind the `gateway_v2` test category. +/// standard-gateway traffic. This cross-region test still requires a +/// multi-region Gateway 2.0 account. #[tokio::test] #[cfg_attr( not(any( test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires a live multi-region Gateway 2.0 account" )] pub async fn gateway_v2_request_timeout_cross_region_for_reads() -> Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() @@ -507,16 +502,15 @@ pub async fn gateway_v2_request_timeout_cross_region_for_reads() -> Result<(), B /// would be a wasted metadata round-trip. /// /// The rule is scoped to [`TransportKind::GatewayV2`] so it does not also -/// fire on standard-gateway requests. The emulator does not yet expose -/// Gateway 2.0 endpoints, so this test is gated behind the `gateway_v2` -/// test category until CI gains a Gateway 2.0 account. +/// fire on standard-gateway requests. This remote-preferred test still +/// requires a multi-region Gateway 2.0 account. #[tokio::test] #[cfg_attr( not(any( test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires a live multi-region Gateway 2.0 account" )] pub async fn gateway_v2_read_session_not_available_remote_preferred() -> Result<(), Box> { @@ -584,17 +578,16 @@ pub async fn gateway_v2_read_session_not_available_remote_preferred() -> Result< /// (id `0x0015`, type `Double`) surfacing in `CosmosResponse::headers()` /// proves the parser resumed correctly after the skip. /// -/// The Cosmos DB emulator does not support Gateway 2.0, so this test is -/// gated behind the `gateway_v2` test category and requires a real -/// Gateway 2.0-enabled account in CI; the SDK in-memory emulator does not -/// implement Gateway 2.0 either. +/// This protocol test runs against either the hosted emulator or a live +/// Gateway 2.0 account. #[tokio::test] #[cfg_attr( not(any( + test_category = "emulator_inmemory_gateway_v2", test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires hosted or live single-region Gateway 2.0" )] pub async fn gateway_v2_unknown_rntbd_response_token_is_silently_skipped( ) -> Result<(), Box> { @@ -760,10 +753,11 @@ fn build_rntbd_response_with_unknown_token(request_charge: f64, body: &[u8]) -> #[tokio::test] #[cfg_attr( not(any( + test_category = "emulator_inmemory_gateway_v2", test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires hosted or live single-region Gateway 2.0" )] pub async fn gateway_v2_server_response_delay_is_injected() -> Result<(), Box> { const INJECTED_DELAY: Duration = Duration::from_millis(500); @@ -828,10 +822,11 @@ pub async fn gateway_v2_server_response_delay_is_injected() -> Result<(), Box Result<(), Box> { const HIT_LIMIT: u32 = 2; @@ -984,10 +979,11 @@ pub async fn fault_injection_449_retry_with_succeeds_after_hit_limit() -> Result #[tokio::test] #[cfg_attr( not(any( + test_category = "emulator_inmemory_gateway_v2", test_category = "gateway_v2", test_category = "gateway_v2_multi_region" )), - ignore = "requires test_category 'gateway_v2'" + ignore = "requires hosted or live single-region Gateway 2.0" )] pub async fn gateway_v2_449_retry_with_succeeds_after_hit_limit() -> Result<(), Box> { let condition = FaultInjectionConditionBuilder::new() diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs index 3ae6b1d00c0..591f2907207 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs @@ -46,7 +46,7 @@ async fn create_duplicate_409() { } #[tokio::test] -async fn upsert_missing_with_if_match_returns_412() { +async fn upsert_missing_with_if_match_creates_item() { let ctx = setup_single_region().await; let body = serde_json::json!({"id": "missing", "pk": "pk1", "value": 42}); let mut request = create_item_request( @@ -65,7 +65,7 @@ async fn upsert_missing_with_if_match_returns_412() { .insert(IF_MATCH.clone(), HeaderValue::from_static("\"stale\"")); let response = ctx.emulator.execute_request(&request).await.unwrap(); - assert_eq!(response.status(), StatusCode::PreconditionFailed); + assert_eq!(response.status(), StatusCode::Created); } #[tokio::test] diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs index db9ad4764db..3d24f734dfc 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs @@ -122,7 +122,7 @@ pub struct MultiRegionTestContext { // Reuse the response-builder header constants from the emulator itself so // tests cannot drift from production strings. pub use azure_data_cosmos_driver::in_memory_emulator::test_headers::{ - ACTIVITY_ID, ETAG, REQUEST_CHARGE, SESSION_TOKEN, SUBSTATUS, + ACTIVITY_ID, ETAG, ITEM_LSN, LSN, REQUEST_CHARGE, SESSION_TOKEN, SUBSTATUS, }; // Request-side headers are only set by tests, so they live here. @@ -131,6 +131,7 @@ pub static IS_UPSERT: HeaderName = HeaderName::from_static("x-ms-documentdb-is-u pub static CONTENT_RESPONSE: HeaderName = HeaderName::from_static("x-ms-cosmos-populate-content-response-on-write"); pub static IF_MATCH: HeaderName = HeaderName::from_static("if-match"); +pub static IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match"); /// Helper to create a POST request to create a document. pub fn create_item_request( diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/point_operations.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/point_operations.rs index 45246990ca7..8536cac5c73 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/point_operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/point_operations.rs @@ -70,6 +70,69 @@ async fn read_existing_item() { assert!(doc.get("_etag").is_some()); } +#[tokio::test] +async fn conditional_read_reports_item_lsn_not_partition_lsn() { + let ctx = setup_single_region().await; + + let first = serde_json::json!({"id": "item1", "pk": "pk1", "value": 1}); + let response = ctx + .emulator + .execute_request(&create_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &first, + r#"["pk1"]"#, + true, + )) + .await + .unwrap(); + let (_, first_headers, first_body) = collect_response(response).await; + let first_etag = first_body["_etag"].as_str().unwrap().to_owned(); + let first_item_lsn = first_headers + .get_optional_str(&ITEM_LSN) + .unwrap() + .to_owned(); + + let second = serde_json::json!({"id": "item2", "pk": "pk1", "value": 2}); + let response = ctx + .emulator + .execute_request(&create_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &second, + r#"["pk1"]"#, + false, + )) + .await + .unwrap(); + let (_, second_headers, _) = collect_response(response).await; + assert_ne!( + second_headers.get_optional_str(&LSN), + Some(first_item_lsn.as_str()), + "the second write must advance the partition LSN" + ); + + let mut request = read_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + "item1", + r#"["pk1"]"#, + ); + request + .headers_mut() + .insert(IF_NONE_MATCH.clone(), HeaderValue::from(first_etag)); + let response = ctx.emulator.execute_request(&request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::NotModified); + assert_eq!( + response.headers().get_optional_str(&ITEM_LSN), + Some(first_item_lsn.as_str()) + ); +} + #[tokio::test] async fn replace_existing_item() { let ctx = setup_single_region().await; diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/split_merge.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/split_merge.rs index 19f38a3aeb5..ad45ef4a942 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/split_merge.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/split_merge.rs @@ -6,9 +6,38 @@ //! Partition split and merge integration tests. use super::*; -use azure_core::http::{Method, Request, Url}; +use azure_core::http::{headers::HeaderValue, Method, Request, Url}; use std::time::Duration; +async fn setup_delayed_multi_region() -> MultiRegionTestContext { + let east_url = "https://eastus.emulator.local"; + let west_url = "https://westus.emulator.local"; + let config = VirtualAccountConfig::new(vec![ + VirtualRegion::new("East US", Url::parse(east_url).unwrap()), + VirtualRegion::new("West US", Url::parse(west_url).unwrap()), + ]) + .unwrap() + .with_write_mode(WriteMode::Single) + .with_consistency(ConsistencyLevel::Session) + .with_replication_config(ReplicationConfig::fixed(Duration::from_millis(50))); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(config)); + let store = emulator.store(); + store.create_database("testdb"); + store.create_container( + "testdb", + "testcoll", + serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(), + ); + MultiRegionTestContext { + emulator, + east_url: east_url.to_owned(), + west_url: west_url.to_owned(), + } +} + #[tokio::test] async fn split_creates_two_children() { let ctx = setup_single_region().await; @@ -119,6 +148,107 @@ async fn split_locked_returns_410_1007() { assert_eq!(substatus, "1007"); } +#[tokio::test] +async fn replication_survives_manual_split_lock() { + let ctx = setup_delayed_multi_region().await; + let store = ctx.emulator.store(); + let replicated = serde_json::json!({"id": "replicated", "pk": "locked", "value": 1}); + let response = ctx + .emulator + .execute_request(&create_item_request( + &ctx.east_url, + "testdb", + "testcoll", + &replicated, + r#"["locked"]"#, + false, + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::Created); + let target_partition = response + .headers() + .get_optional_str(&SESSION_TOKEN) + .and_then(|token| token.split(':').next()) + .and_then(|value| value.parse::().ok()) + .unwrap(); + let split_epk = store + .midpoint_split_epk("testdb", "testcoll", target_partition) + .unwrap(); + let operation = + store.begin_manual_split_partition("testdb", "testcoll", target_partition, split_epk); + store.drain_pending_replications().await; + operation.complete().await.unwrap(); + + let response = ctx + .emulator + .execute_request(&read_item_request( + &ctx.east_url, + "testdb", + "testcoll", + "replicated", + r#"["locked"]"#, + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::Ok); + assert_eq!(store.dropped_replications(), 0); +} + +#[tokio::test] +async fn replication_survives_manual_merge_lock() { + let ctx = setup_delayed_multi_region().await; + let store = ctx.emulator.store(); + + let mut routed = None; + for index in 0..256 { + let pk = format!("merge-{index}"); + let body = serde_json::json!({"id": "replicated", "pk": pk, "value": index}); + let pk_header = serde_json::json!([pk]).to_string(); + let response = ctx + .emulator + .execute_request(&create_item_request( + &ctx.east_url, + "testdb", + "testcoll", + &body, + &pk_header, + false, + )) + .await + .unwrap(); + let partition_id = response + .headers() + .get_optional_str(&SESSION_TOKEN) + .and_then(|token| token.split(':').next()) + .and_then(|value| value.parse::().ok()) + .unwrap(); + if partition_id == 0 || partition_id == 1 { + routed = Some((pk_header, partition_id)); + break; + } + store.drain_pending_replications().await; + } + let (pk_header, _) = routed.expect("expected an item routed to partition 0 or 1"); + let operation = store.begin_manual_merge_partitions("testdb", "testcoll", 0, 1); + store.drain_pending_replications().await; + operation.complete().await.unwrap(); + + let response = ctx + .emulator + .execute_request(&read_item_request( + &ctx.west_url, + "testdb", + "testcoll", + "replicated", + &pk_header, + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::Ok); + assert_eq!(store.dropped_replications(), 0); +} + #[tokio::test] async fn split_preserves_vector_clock_version() { let ctx = setup_single_region().await; @@ -188,6 +318,56 @@ async fn merge_adjacent_partitions() { assert!(parents.contains(&serde_json::json!("1"))); } +#[tokio::test] +async fn merge_advances_pkrange_feed_etag() { + let ctx = setup_single_region().await; + let store = ctx.emulator.store(); + let url = format!("{}/dbs/testdb/colls/testcoll/pkranges", ctx.gateway_url); + + let response = ctx + .emulator + .execute_request(&Request::new(Url::parse(&url).unwrap(), Method::Get)) + .await + .unwrap(); + let (_, initial_headers, initial_body) = collect_response(response).await; + let initial_etag = initial_headers.get_optional_str(&ETAG).unwrap().to_owned(); + assert_eq!( + initial_body["PartitionKeyRanges"].as_array().unwrap().len(), + 4 + ); + + store.merge_partitions("testdb", "testcoll", 0, 1, Duration::ZERO); + store.drain_pending_control_plane().await; + + let mut refresh = Request::new(Url::parse(&url).unwrap(), Method::Get); + refresh.headers_mut().insert( + IF_NONE_MATCH.clone(), + HeaderValue::from(initial_etag.clone()), + ); + let response = ctx.emulator.execute_request(&refresh).await.unwrap(); + let (status, refreshed_headers, refreshed_body) = collect_response(response).await; + assert_eq!(status, StatusCode::Ok); + let refreshed_etag = refreshed_headers + .get_optional_str(&ETAG) + .unwrap() + .to_owned(); + assert_ne!(refreshed_etag, initial_etag); + assert_eq!( + refreshed_body["PartitionKeyRanges"] + .as_array() + .unwrap() + .len(), + 3 + ); + + let mut not_modified = Request::new(Url::parse(&url).unwrap(), Method::Get); + not_modified + .headers_mut() + .insert(IF_NONE_MATCH.clone(), HeaderValue::from(refreshed_etag)); + let response = ctx.emulator.execute_request(¬_modified).await.unwrap(); + assert_eq!(response.status(), StatusCode::NotModified); +} + #[tokio::test] async fn merge_rejects_non_adjacent_partitions() { let ctx = setup_single_region().await; diff --git a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml index 9e8c2b5852b..65156bee51f 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = [ "fs", + "io-util", "net", "rt-multi-thread", "signal", diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md index b067a2f4007..e3f40cc1d8d 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/adr/001_build_memory_backed_sdk_test_emulator.md @@ -16,8 +16,10 @@ provide the fine-grained, runtime topology controls required by SDK routing and They also introduce platform, container, startup-time, and certificate dependencies that make large deterministic test matrices more expensive and less predictable. -A network-accessible host is required so Rust, Java, .NET, Python, and other SDKs can exercise their -real HTTP stacks and wire protocols against the same controllable topology model. +A network-accessible host is required so SDKs can eventually exercise their real HTTP stacks and +wire protocols against the same controllable topology model. Initial Gateway 2.0 support targets +Rust and generic h2c-capable clients; stock peer-SDK compatibility requires independent validation +and, where necessary, the deferred TLS/H2 listener. ## Decision @@ -48,8 +50,8 @@ are added when they enable a concrete SDK test scenario, not to pursue general C parity. Tests can run quickly and deterministically without Azure resources or heavyweight emulator -infrastructure. Cross-language SDKs can share the same network-visible behavior while retaining -their real client pipelines. +infrastructure. Cross-language SDKs can adopt the same network-visible behavior after validating +their transport requirements and wire compatibility independently. The project must document supported behavior and known divergences clearly so its intentionally limited scope is not mistaken for customer-grade service emulation. diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index 276311f7fe8..e8864871373 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -203,9 +203,14 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-006. | `databases[].containers[].throughput` | u32 | Provisioned RU/s (drives throttling when enabled). | | `databases[].containers[].seedItems[]` | array | Documents created on startup; each carries its `partitionKey` value array and the `document` body. | +Region names and effective region IDs must be unique; region-name references are case-sensitive. +Every explicit nonzero listener port must also be unique. Database IDs must be unique, container +IDs must be unique within a database, and resource IDs may not be empty or contain `/`, `\`, `?`, +or `#`. The host validates the complete configuration before provisioning any resource. + Seed items are created through the same request path as real writes (a synthesized create-item request per item), so EPK routing, RU accounting, and replication behave identically to -client-issued writes. +client-issued writes. The host waits for all scheduled seed replication before publishing ready. After binding all listeners, the host writes one JSON `ready` record to stdout. It contains the resolved management endpoint, hub account endpoint, and all regional standard-gateway and @@ -249,6 +254,10 @@ Enabled per region by setting `gateway20Port`. When enabled: the RNTBD request frame + literal `thinclient` headers, reconstructs the logical operation, dispatches it through the same store, and **encodes** the result as an RNTBD response frame. +PR1 validates Gateway 2.0 with the Rust SDK and generic clients that support cleartext HTTP/2 +(`h2c`). Stock Java Gateway 2.0 compatibility requires the deferred TLS/H2 listener; peer-SDK +interoperability is a validation target, not a compatibility guarantee of this phase. + The driver already owns the client-side RNTBD codec (`RntbdRequestFrame::write`, `RntbdResponse::read`). This work promotes the currently test-only inverse halves (`RntbdRequestFrame::read`, `RntbdResponse::write`) to production, co-located in @@ -295,7 +304,7 @@ POST /databases/{db}/containers/{coll}/partitions/{partitionId}/split "mode": "midpoint" | "epk" | "storage", // default: "midpoint" "epk": "", // required when mode = "epk"; ignored otherwise "progressionMode": "automatic" | "manual", // default: "automatic" - "lockDurationMs": 500 // automatic only; default: 0 + "lockDurationMs": 500 // automatic only; default: 0; maximum: 60000 } → 202 { "operationId": "op-split-123", "status": "Running", "phase": "Preparing" } diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs index d07939c49c7..4b927df331e 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs @@ -143,10 +143,37 @@ pub(crate) struct GatewayBinding { pub(crate) gateway20_url: Option, } +pub(crate) struct BoundEndpoint { + pub(crate) url: Url, + pub(crate) listener: TcpListener, +} + pub(crate) struct BoundGateway { - pub(crate) binding: GatewayBinding, - pub(crate) gateway_listener: TcpListener, - pub(crate) gateway20_listener: Option, + pub(crate) region_name: String, + pub(crate) gateway: BoundEndpoint, + pub(crate) gateway20: Option, +} + +impl BoundGateway { + pub(crate) fn binding(&self) -> GatewayBinding { + GatewayBinding { + region_name: self.region_name.clone(), + gateway_url: self.gateway.url.clone(), + gateway20_url: self.gateway20.as_ref().map(|endpoint| endpoint.url.clone()), + } + } +} + +pub(crate) struct BoundHost { + pub(crate) gateways: Vec, + pub(crate) management: BoundEndpoint, +} + +#[derive(Clone, Copy)] +enum ListenerSlot { + Gateway(usize), + Gateway20(usize), + Management, } impl EmulatorConfig { @@ -157,33 +184,61 @@ impl EmulatorConfig { Ok(config) } - pub(crate) async fn bind_gateways(&self) -> Result> { - let mut gateways = Vec::with_capacity(self.account.regions.len()); - for region in &self.account.regions { - let gateway_listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], region.gateway_port))).await?; - let gateway_url = loopback_url(gateway_listener.local_addr()?.port())?; - let gateway20_listener = match region.gateway20_port { - Some(port) => { - Some(TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))).await?) - } - None => None, + pub(crate) async fn bind(&self) -> Result { + let mut specifications = Vec::new(); + for (index, region) in self.account.regions.iter().enumerate() { + specifications.push((ListenerSlot::Gateway(index), region.gateway_port)); + if let Some(port) = region.gateway20_port { + specifications.push((ListenerSlot::Gateway20(index), port)); + } + } + specifications.push((ListenerSlot::Management, self.management.port)); + specifications.sort_by_key(|(_, port)| *port == 0); + + let region_count = self.account.regions.len(); + let mut gateways: Vec> = + std::iter::repeat_with(|| None).take(region_count).collect(); + let mut gateways20: Vec> = + std::iter::repeat_with(|| None).take(region_count).collect(); + let mut management = None; + for (slot, port) in specifications { + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))).await?; + let local_address = listener.local_addr()?; + if !local_address.ip().is_loopback() { + return Err( + format!("emulator listener must bind to loopback: {local_address}").into(), + ); + } + let endpoint = BoundEndpoint { + url: loopback_url(local_address.port())?, + listener, }; - let gateway20_url = gateway20_listener - .as_ref() - .map(|listener| loopback_url(listener.local_addr()?.port())) - .transpose()?; - gateways.push(BoundGateway { - binding: GatewayBinding { - region_name: region.name.clone(), - gateway_url, - gateway20_url, - }, - gateway_listener, - gateway20_listener, - }); + match slot { + ListenerSlot::Gateway(index) => gateways[index] = Some(endpoint), + ListenerSlot::Gateway20(index) => gateways20[index] = Some(endpoint), + ListenerSlot::Management => management = Some(endpoint), + } } - Ok(gateways) + + let gateways = self + .account + .regions + .iter() + .enumerate() + .map(|(index, region)| { + Ok(BoundGateway { + region_name: region.name.clone(), + gateway: gateways[index] + .take() + .ok_or("configured gateway listener was not bound")?, + gateway20: gateways20[index].take(), + }) + }) + .collect::>>()?; + Ok(BoundHost { + gateways, + management: management.ok_or("management listener was not bound")?, + }) } pub(crate) fn create_emulator( @@ -202,7 +257,7 @@ impl EmulatorConfig { let mut virtual_region = VirtualRegion::new(®ion.name, binding.gateway_url.clone()); if let Some(gateway20_url) = &binding.gateway20_url { - virtual_region = virtual_region.with_thin_client_url(gateway20_url.clone()); + virtual_region = virtual_region.with_gateway_v2_url(gateway20_url.clone()); } match region.region_id { Some(region_id) => virtual_region.with_region_id(region_id), @@ -245,6 +300,7 @@ impl EmulatorConfig { emulator: &Arc, gateway_url: &Url, ) -> Result<()> { + self.validate()?; let store = emulator.store(); for database in &self.databases { @@ -274,6 +330,7 @@ impl EmulatorConfig { } } } + store.drain_pending_replications().await; Ok(()) } @@ -316,10 +373,49 @@ impl EmulatorConfig { } } } + let mut database_ids = std::collections::HashSet::new(); + for database in &self.databases { + validate_resource_id("database", &database.id)?; + if !database_ids.insert(database.id.as_str()) { + return Err( + format!("database '{}' is configured more than once", database.id).into(), + ); + } + let mut container_ids = std::collections::HashSet::new(); + for container in &database.containers { + validate_resource_id("container", &container.id)?; + if !container_ids.insert(container.id.as_str()) { + return Err(format!( + "container '{}/{}' is configured more than once", + database.id, container.id + ) + .into()); + } + let mut container_config = + ContainerConfig::new().with_partition_count(container.partition_count); + if let Some(throughput) = container.throughput { + container_config = container_config.with_throughput(throughput); + } + container_config.build()?; + } + } Ok(()) } } +fn validate_resource_id(resource_type: &str, id: &str) -> Result<()> { + if id.trim().is_empty() { + return Err(format!("{resource_type} IDs must not be empty").into()); + } + if id.contains('/') || id.contains('\\') || id.contains('?') || id.contains('#') { + return Err(format!( + "{resource_type} ID '{id}' contains a character that cannot be used in a Cosmos resource path" + ) + .into()); + } + Ok(()) +} + async fn seed_document( emulator: &InMemoryEmulatorHttpClient, gateway_url: &Url, @@ -327,7 +423,10 @@ async fn seed_document( container_id: &str, seed_item: &SeedItem, ) -> Result<()> { - let url = gateway_url.join(&format!("dbs/{database_id}/colls/{container_id}/docs"))?; + let mut url = gateway_url.clone(); + url.path_segments_mut() + .map_err(|_| "gateway URL cannot be used as a base URL")? + .extend(["dbs", database_id, "colls", container_id, "docs"]); let mut request = Request::new(url, Method::Post); request.headers_mut().insert( "x-ms-documentdb-partitionkey", @@ -433,10 +532,11 @@ mod tests { })) .unwrap(); config.validate().unwrap(); - let bound_gateways = config.bind_gateways().await.unwrap(); - let bindings: Vec<_> = bound_gateways + let bound_host = config.bind().await.unwrap(); + let bindings: Vec<_> = bound_host + .gateways .iter() - .map(|gateway| gateway.binding.clone()) + .map(|gateway| gateway.binding()) .collect(); let gateway_url = bindings[0].gateway_url.clone(); let emulator = config.create_emulator(&bindings).unwrap(); @@ -486,19 +586,36 @@ mod tests { zero_port.validate().unwrap(); assert_eq!(zero_port.management.port, 0); - let gateways = zero_port.bind_gateways().await.unwrap(); - let gateway = &gateways[0]; - let gateway_port = gateway.binding.gateway_url.port().unwrap(); - let gateway20_port = gateway - .binding - .gateway20_url - .as_ref() - .unwrap() - .port() - .unwrap(); + let host = zero_port.bind().await.unwrap(); + let gateway = &host.gateways[0]; + let gateway_port = gateway.gateway.url.port().unwrap(); + let gateway20_port = gateway.gateway20.as_ref().unwrap().url.port().unwrap(); assert_ne!(gateway_port, 0); assert_ne!(gateway20_port, 0); assert_ne!(gateway_port, gateway20_port); + assert!(gateway + .gateway + .listener + .local_addr() + .unwrap() + .ip() + .is_loopback()); + assert!(gateway + .gateway20 + .as_ref() + .unwrap() + .listener + .local_addr() + .unwrap() + .ip() + .is_loopback()); + assert!(host + .management + .listener + .local_addr() + .unwrap() + .ip() + .is_loopback()); } #[test] @@ -517,4 +634,75 @@ mod tests { .unwrap(); assert!(duplicate.validate().is_err()); } + + #[test] + fn rejects_duplicate_startup_resources() { + let duplicate_database: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US" }] + }, + "databases": [{ "id": "db" }, { "id": "db" }] + })) + .unwrap(); + assert!(duplicate_database.validate().is_err()); + + let duplicate_container: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US" }] + }, + "databases": [{ + "id": "db", + "containers": [ + { "id": "coll", "partitionKey": { "paths": ["/pk"], "kind": "Hash", "version": 2 } }, + { "id": "coll", "partitionKey": { "paths": ["/pk"], "kind": "Hash", "version": 2 } } + ] + }] + })) + .unwrap(); + assert!(duplicate_container.validate().is_err()); + } + + #[tokio::test] + async fn invalid_startup_config_does_not_partially_provision() { + let config: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "Session", + "regions": [{ "name": "East US" }] + }, + "databases": [ + { "id": "valid" }, + { + "id": "invalid", + "containers": [{ + "id": "coll", + "partitionKey": { "paths": ["/pk"], "kind": "Hash", "version": 2 }, + "partitionCount": 0 + }] + } + ] + })) + .unwrap(); + let bindings = vec![GatewayBinding { + region_name: "East US".to_owned(), + gateway_url: Url::parse("http://127.0.0.1:18081/").unwrap(), + gateway20_url: None, + }]; + let emulator = config.create_emulator(&bindings).unwrap(); + let result = config.provision(&emulator, &bindings[0].gateway_url).await; + assert!(result.is_err()); + let request = Request::new( + bindings[0].gateway_url.join("dbs/valid").unwrap(), + Method::Get, + ); + let response = emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), azure_core::http::StatusCode::NotFound); + } } diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs index c1828c2b65f..481b4e9b29a 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs @@ -82,9 +82,14 @@ pub(crate) async fn into_cosmos_request( .path_and_query() .map(|value| value.as_str()) .unwrap_or("/"); - let url = base_url - .join(path_and_query.trim_start_matches('/')) - .map_err(internal_error)?; + if path_and_query.starts_with("//") { + return Err(( + StatusCode::BAD_REQUEST, + "request paths must contain exactly one leading slash".to_owned(), + )); + } + let relative_path = path_and_query.strip_prefix('/').unwrap_or(path_and_query); + let url = base_url.join(relative_path).map_err(internal_error)?; let bytes = to_bytes(body, MAX_REQUEST_BODY_SIZE) .await .map_err(|error| (StatusCode::PAYLOAD_TOO_LARGE, error.to_string()))?; @@ -147,4 +152,16 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); } + + #[tokio::test] + async fn rejects_multi_slash_paths_without_replacing_authority() { + let base_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let request = Request::builder() + .uri("//attacker.example/dbs") + .body(Body::empty()) + .unwrap(); + + let error = into_cosmos_request(request, &base_url).await.unwrap_err(); + assert_eq!(error.0, StatusCode::BAD_REQUEST); + } } diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs index 1446e887ec6..04b72c55bd8 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/gateway_v2.rs @@ -19,7 +19,7 @@ use serde_json::json; use tokio::net::TcpListener; use url::Url; -use crate::{config::GatewayBinding, data_plane, metrics::HostMetrics}; +use crate::{data_plane, metrics::HostMetrics}; #[derive(Clone)] struct GatewayV2State { @@ -34,15 +34,13 @@ struct GatewayV2State { pub(crate) async fn serve( listener: TcpListener, - binding: GatewayBinding, + region_name: String, + base_url: Url, emulator: Arc, metrics: Arc, ) -> io::Result<()> { - let Some(base_url) = binding.gateway20_url else { - return Ok(()); - }; tracing::info!( - region = binding.region_name, + region = region_name, endpoint = %base_url, "Cosmos Gateway 2.0 listener ready" ); @@ -87,8 +85,13 @@ async fn execute( "Gateway 2.0 requires HTTP/2".to_owned(), )); } - if request.method() == axum::http::Method::POST && request.uri().path() == "/connectivity-probe" - { + if request.method() != axum::http::Method::POST { + return Err(( + StatusCode::METHOD_NOT_ALLOWED, + "Gateway 2.0 requires POST".to_owned(), + )); + } + if request.uri().path() == "/connectivity-probe" { state.metrics.record_connectivity_probe(); #[cfg(test)] if let Some(probe_count) = &state.probe_count { @@ -99,19 +102,23 @@ async fn execute( .body(Body::empty()) .map_err(data_plane::internal_error); } - state.metrics.record_gateway20_request(); - #[cfg(test)] - if let Some(request_count) = &state.request_count { - request_count.fetch_add(1, Ordering::SeqCst); - } - let request = data_plane::into_cosmos_request(request, &state.base_url).await?; let response = state .emulator .execute_gateway_v2_request(&request) .await - .map_err(data_plane::internal_error)?; - data_plane::into_http_response(response).await + .map_err(|error| { + let status = StatusCode::from_u16(u16::from(error.status().status_code())) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + (status, error.to_string()) + })?; + let response = data_plane::into_http_response(response).await?; + state.metrics.record_gateway20_request(); + #[cfg(test)] + if let Some(request_count) = &state.request_count { + request_count.fetch_add(1, Ordering::SeqCst); + } + Ok(response) } #[cfg(test)] @@ -131,7 +138,7 @@ mod tests { async fn connectivity_probe_requires_http2_and_returns_ok() { let base_url = Url::parse("http://127.0.0.1:18444/").unwrap(); let region = VirtualRegion::new("East US", Url::parse("http://127.0.0.1:18081/").unwrap()) - .with_thin_client_url(base_url.clone()); + .with_gateway_v2_url(base_url.clone()); let emulator = Arc::new(InMemoryEmulatorHttpClient::new( VirtualAccountConfig::new(vec![region]).unwrap(), )); @@ -161,6 +168,33 @@ mod tests { assert_eq!(error.0, StatusCode::HTTP_VERSION_NOT_SUPPORTED); } + #[tokio::test] + async fn data_requests_require_post() { + let base_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", Url::parse("http://127.0.0.1:18081/").unwrap()) + .with_gateway_v2_url(base_url.clone()); + let state = GatewayV2State { + emulator: Arc::new(InMemoryEmulatorHttpClient::new( + VirtualAccountConfig::new(vec![region]).unwrap(), + )), + base_url, + metrics: Arc::new(HostMetrics::default()), + request_count: None, + probe_count: None, + }; + + for method in ["GET", "PUT", "DELETE"] { + let mut request = Request::builder() + .method(method) + .uri("/") + .body(Body::empty()) + .unwrap(); + *request.version_mut() = Version::HTTP_2; + let error = execute(state.clone(), request).await.unwrap_err(); + assert_eq!(error.0, StatusCode::METHOD_NOT_ALLOWED); + } + } + #[tokio::test] async fn real_driver_uses_hosted_gateway20_over_h2c() { let gateway_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -173,7 +207,7 @@ mod tests { let thin_url = Url::parse(&format!("http://{}/", thin_listener.local_addr().unwrap())).unwrap(); let region = VirtualRegion::new("East US", gateway_url.clone()) - .with_thin_client_url(thin_url.clone()); + .with_gateway_v2_url(thin_url.clone()); let emulator = Arc::new(InMemoryEmulatorHttpClient::new( VirtualAccountConfig::new(vec![region]).unwrap(), )); diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs index f7b540accbf..a4a51b0f3d1 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/main.rs @@ -9,7 +9,6 @@ mod metrics; use std::{ io::{self, Write}, - net::SocketAddr, path::PathBuf, }; @@ -17,7 +16,6 @@ use clap::Parser; use config::{EmulatorConfig, GatewayBinding}; use metrics::HostMetrics; use serde::Serialize; -use tokio::net::TcpListener; use url::Url; type Result = std::result::Result>; @@ -42,14 +40,13 @@ async fn main() -> Result<()> { let args = Args::parse(); let config = EmulatorConfig::load(&args.config).await?; - let bound_gateways = config.bind_gateways().await?; - let bindings: Vec<_> = bound_gateways + let bound_host = config.bind().await?; + let bindings: Vec<_> = bound_host + .gateways .iter() - .map(|gateway| gateway.binding.clone()) + .map(|gateway| gateway.binding()) .collect(); - let management_listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], config.management.port))).await?; - let management_endpoint = loopback_url(management_listener.local_addr()?.port())?; + let management_endpoint = bound_host.management.url.clone(); let account_endpoint = bindings .first() .ok_or("no emulator gateway listeners were configured")? @@ -60,23 +57,30 @@ async fn main() -> Result<()> { let metrics = std::sync::Arc::new(HostMetrics::default()); let mut listeners = tokio::task::JoinSet::new(); - for bound_gateway in bound_gateways { - let binding = bound_gateway.binding; + for bound_gateway in bound_host.gateways { + let binding = bound_gateway.binding(); let gateway_emulator = emulator.clone(); let gateway_binding = binding.clone(); listeners.spawn(async move { data_plane::serve( - bound_gateway.gateway_listener, + bound_gateway.gateway.listener, gateway_binding, gateway_emulator, ) .await }); - if let Some(gateway20_listener) = bound_gateway.gateway20_listener { + if let Some(gateway20) = bound_gateway.gateway20 { let gateway20_emulator = emulator.clone(); let metrics = metrics.clone(); listeners.spawn(async move { - gateway_v2::serve(gateway20_listener, binding, gateway20_emulator, metrics).await + gateway_v2::serve( + gateway20.listener, + binding.region_name, + gateway20.url, + gateway20_emulator, + metrics, + ) + .await }); } } @@ -84,7 +88,7 @@ async fn main() -> Result<()> { let management_bindings = bindings.clone(); listeners.spawn(async move { management::serve( - management_listener, + bound_host.management.listener, emulator, account_id, management_bindings, @@ -148,7 +152,3 @@ fn write_ready_record( stdout.flush()?; Ok(()) } - -fn loopback_url(port: u16) -> Result { - Ok(Url::parse(&format!("http://127.0.0.1:{port}/"))?) -} diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs index 050371d156e..dccac8f27d5 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs @@ -14,7 +14,8 @@ use std::{ }; use axum::{ - extract::{Path, State}, + body::to_bytes, + extract::{Path, Request, State}, http::StatusCode, response::{IntoResponse, Response}, routing::{get, post, put}, @@ -30,6 +31,7 @@ use tokio::{net::TcpListener, sync::Mutex}; use crate::{config::GatewayBinding, metrics::HostMetrics}; const MAX_CONTROL_PLANE_LOCK_DURATION_MS: u64 = 60_000; +const MAX_MANAGEMENT_BODY_SIZE: usize = 1024 * 1024; #[derive(Clone)] struct ManagementState { @@ -264,7 +266,13 @@ fn operation_response(operation_id: &str, operation: &OperationRecord) -> serde_ .as_object_mut() .expect("operation response must be an object"); if let Some(serde_json::Value::Object(result)) = &operation.result { - object.extend(result.clone()); + for (key, value) in result { + assert!( + !matches!(key.as_str(), "operationId" | "status" | "phase" | "error"), + "operation result field '{key}' conflicts with the response envelope" + ); + object.insert(key.clone(), value.clone()); + } } if let Some(error) = &operation.error { object.insert("error".to_owned(), error.clone().into()); @@ -314,9 +322,19 @@ struct SplitOperationDetails { async fn split_partition( State(state): State, Path((database, container, partition_id)): Path<(String, String, u32)>, - request: Option>, + request: Request, +) -> ApiResult<(StatusCode, Json)> { + let request = parse_optional_json(request).await?; + split_partition_inner(state, database, container, partition_id, request).await +} + +async fn split_partition_inner( + state: ManagementState, + database: String, + container: String, + partition_id: u32, + request: SplitRequest, ) -> ApiResult<(StatusCode, Json)> { - let request = request.map(|request| request.0).unwrap_or_default(); let lock_duration = validate_progression(request.progression_mode, request.lock_duration_ms)?; let store = state.emulator.store(); let split_epk = match request.mode { @@ -326,7 +344,9 @@ async fn split_partition( .epk .as_deref() .ok_or_else(|| ApiError::bad_request("epk is required when mode is 'epk'"))?; - parse_epk(value)? + let split_epk = parse_epk(value)?; + store.validate_split_epk(&database, &container, partition_id, &split_epk)?; + split_epk } SplitMode::Storage => store.storage_split_epk(&database, &container, partition_id)?, }; @@ -368,6 +388,41 @@ async fn split_partition( Ok((StatusCode::ACCEPTED, Json(response))) } +async fn parse_optional_json(request: Request) -> ApiResult +where + T: Default + for<'de> Deserialize<'de>, +{ + let content_type = request + .headers() + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body = to_bytes(request.into_body(), MAX_MANAGEMENT_BODY_SIZE) + .await + .map_err(|error| ApiError::bad_request(error.to_string()))?; + if body.is_empty() { + return Ok(T::default()); + } + if !content_type.as_deref().is_some_and(is_json_content_type) { + return Err(ApiError::unsupported_media_type( + "nonempty request bodies require application/json", + )); + } + serde_json::from_slice(&body) + .map_err(|error| ApiError::bad_request(format!("invalid JSON body: {error}"))) +} + +fn is_json_content_type(value: &str) -> bool { + let media_type = value + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + media_type == "application/json" + || (media_type.starts_with("application/") && media_type.ends_with("+json")) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct MergeRequest { @@ -436,31 +491,28 @@ fn spawn_automatic_split( split_epk, } = details; let store = state.emulator.store(); - let operation = - store.begin_manual_split_partition(&database, &container, parent, split_epk.clone()); - set_operation_swapping(&state.operations, &operation_id).await; + let operation = { + let mut operations = state.operations.records.lock().await; + let operation = store.begin_manual_split_partition( + &database, + &container, + parent, + split_epk.clone(), + ); + if let Some(record) = operations.get_mut(&operation_id) { + record.phase = OperationPhase::Swapping; + } + operation + }; if !lock_duration.is_zero() { tokio::time::sleep(lock_duration).await; } - let outcome = match operation.complete().await { - Ok(()) => { - let children = store.child_partition_ids(&database, &container, &[parent]); - if children.len() == 2 { - Ok(json!(SplitResponse { - database, - container, - parent, - children, - mode, - split_epk: split_epk.to_hex(), - })) - } else { - Err("split completed without producing exactly two child partitions".to_owned()) - } - } - Err(error) => Err(error.to_string()), - }; - finish_operation(&state.operations, &operation_id, outcome).await; + let mut operations = state.operations.records.lock().await; + let outcome = complete_split( + &store, database, container, parent, mode, split_epk, operation, + ) + .await; + finish_operation_locked(&mut operations, &operation_id, outcome); }); } @@ -474,47 +526,34 @@ fn spawn_automatic_merge( ) { tokio::spawn(async move { let store = state.emulator.store(); - let operation = store.begin_manual_merge_partitions( - &database, - &container, - partitions[0], - partitions[1], - ); - set_operation_swapping(&state.operations, &operation_id).await; + let operation = { + let mut operations = state.operations.records.lock().await; + let operation = store.begin_manual_merge_partitions( + &database, + &container, + partitions[0], + partitions[1], + ); + if let Some(record) = operations.get_mut(&operation_id) { + record.phase = OperationPhase::Swapping; + } + operation + }; if !lock_duration.is_zero() { tokio::time::sleep(lock_duration).await; } - let outcome = match operation.complete().await { - Ok(()) => { - let children = store.child_partition_ids(&database, &container, &partitions); - match children.as_slice() { - [merged_child] => Ok(json!(MergeResponse { - merged: partitions, - into: *merged_child, - })), - _ => Err( - "merge completed without producing exactly one child partition".to_owned(), - ), - } - } - Err(error) => Err(error.to_string()), - }; - finish_operation(&state.operations, &operation_id, outcome).await; + let mut operations = state.operations.records.lock().await; + let outcome = complete_merge(&store, database, container, partitions, operation).await; + finish_operation_locked(&mut operations, &operation_id, outcome); }); } -async fn set_operation_swapping(operations: &OperationRegistry, operation_id: &str) { - if let Some(operation) = operations.records.lock().await.get_mut(operation_id) { - operation.phase = OperationPhase::Swapping; - } -} - -async fn finish_operation( - operations: &OperationRegistry, +fn finish_operation_locked( + operations: &mut HashMap, operation_id: &str, outcome: Result, ) { - if let Some(operation) = operations.records.lock().await.get_mut(operation_id) { + if let Some(operation) = operations.get_mut(operation_id) { match outcome { Ok(result) => operation.succeed(result), Err(error) => operation.fail(error), @@ -522,6 +561,59 @@ async fn finish_operation( } } +async fn complete_split( + store: &Arc, + database: String, + container: String, + parent: u32, + mode: SplitMode, + split_epk: Epk, + operation: ManualControlPlaneOperation, +) -> Result { + match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &[parent]); + if children.len() == 2 { + Ok(json!(SplitResponse { + database, + container, + parent, + children, + mode, + split_epk: split_epk.to_hex(), + })) + } else { + Err("split completed without producing exactly two child partitions".to_owned()) + } + } + Err(error) => Err(error.to_string()), + } +} + +async fn complete_merge( + store: &Arc, + database: String, + container: String, + partitions: [u32; 2], + operation: ManualControlPlaneOperation, +) -> Result { + match operation.complete().await { + Ok(()) => { + let children = store.child_partition_ids(&database, &container, &partitions); + match children.as_slice() { + [merged_child] => Ok(json!(MergeResponse { + merged: partitions, + into: *merged_child, + })), + _ => { + Err("merge completed without producing exactly one child partition".to_owned()) + } + } + } + Err(error) => Err(error.to_string()), + } +} + async fn get_operation( State(state): State, Path(operation_id): Path, @@ -554,7 +646,7 @@ async fn advance_operation( State(state): State, Path(operation_id): Path, ) -> ApiResult> { - let completion = { + let (completion, response) = { let mut operations = state.operations.records.lock().await; let operation = operations.get_mut(&operation_id).ok_or_else(|| { ApiError::not_found(format!("operation '{operation_id}' does not exist")) @@ -565,6 +657,7 @@ async fn advance_operation( "operation '{operation_id}' is already terminal" ))); } + let response = operation_response(&operation_id, operation); let action = operation.action.take().ok_or_else(|| { ApiError::conflict(format!( "operation '{operation_id}' is automatic or already advancing" @@ -621,84 +714,63 @@ async fn advance_operation( parent, mode, split_epk, - operation, - } => ManualCompletion::Split { + operation: manual_operation, + } => ( + ManualCompletion::Split { + database, + container, + parent, + mode, + split_epk, + operation: manual_operation, + }, + response, + ), + OperationAction::ActiveMerge { + database, + container, + partitions, + operation: manual_operation, + } => ( + ManualCompletion::Merge { + database, + container, + partitions, + operation: manual_operation, + }, + response, + ), + } + }; + let operations = state.operations.clone(); + let store = state.emulator.store(); + let completion_id = operation_id.clone(); + tokio::spawn(async move { + let mut records = operations.records.lock().await; + let outcome = match completion { + ManualCompletion::Split { database, container, parent, mode, split_epk, operation, - }, - OperationAction::ActiveMerge { - database, - container, - partitions, - operation, - } => ManualCompletion::Merge { + } => { + complete_split( + &store, database, container, parent, mode, split_epk, operation, + ) + .await + } + ManualCompletion::Merge { database, container, partitions, operation, - }, - } - }; - - let store = state.emulator.store(); - let outcome = match completion { - ManualCompletion::Split { - database, - container, - parent, - mode, - split_epk, - operation, - } => match operation.complete().await { - Ok(()) => { - let children = store.child_partition_ids(&database, &container, &[parent]); - if children.len() == 2 { - Ok(json!(SplitResponse { - database, - container, - parent, - children, - mode, - split_epk: split_epk.to_hex(), - })) - } else { - Err("split completed without producing exactly two child partitions".to_owned()) - } - } - Err(error) => Err(error.to_string()), - }, - ManualCompletion::Merge { - database, - container, - partitions, - operation, - } => match operation.complete().await { - Ok(()) => { - let children = store.child_partition_ids(&database, &container, &partitions); - match children.as_slice() { - [merged_child] => Ok(json!(MergeResponse { - merged: partitions, - into: *merged_child, - })), - _ => Err( - "merge completed without producing exactly one child partition".to_owned(), - ), - } - } - Err(error) => Err(error.to_string()), - }, - }; - finish_operation(&state.operations, &operation_id, outcome).await; - - let operations = state.operations.records.lock().await; - let operation = operations - .get(&operation_id) - .ok_or_else(|| ApiError::not_found(format!("operation '{operation_id}' does not exist")))?; - Ok(Json(operation_response(&operation_id, operation))) + } => complete_merge(&store, database, container, partitions, operation).await, + }; + finish_operation_locked(&mut records, &completion_id, outcome); + }); + Ok(Json(response)) } #[derive(Debug, Deserialize, Serialize)] @@ -810,6 +882,13 @@ impl ApiError { message: message.into(), } } + + fn unsupported_media_type(message: impl Into) -> Self { + Self { + status: StatusCode::UNSUPPORTED_MEDIA_TYPE, + message: message.into(), + } + } } impl From for ApiError { @@ -839,6 +918,61 @@ mod tests { }; use url::Url; + async fn http_json( + address: std::net::SocketAddr, + method: &str, + path: &str, + body: Option<&str>, + ) -> (u16, serde_json::Value) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut stream = tokio::net::TcpStream::connect(address).await.unwrap(); + let body = body.unwrap_or_default(); + let content_headers = if body.is_empty() { + String::new() + } else { + format!( + "Content-Type: application/json\r\nContent-Length: {}\r\n", + body.len() + ) + }; + let request = format!( + "{method} {path} HTTP/1.1\r\nHost: {address}\r\n{content_headers}Connection: close\r\n\r\n{body}" + ); + stream.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + let response = String::from_utf8(response).unwrap(); + let (headers, body) = response.split_once("\r\n\r\n").unwrap(); + let status = headers + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .parse() + .unwrap(); + (status, serde_json::from_str(body).unwrap()) + } + + async fn wait_for_phase( + state: &ManagementState, + operation_id: &str, + expected: &str, + ) -> serde_json::Value { + for _ in 0..100 { + let operation = get_operation(State(state.clone()), Path(operation_id.to_owned())) + .await + .unwrap(); + if operation["phase"] == expected { + return operation.0; + } + tokio::task::yield_now().await; + } + panic!("operation {operation_id} did not reach phase {expected}") + } + #[tokio::test] async fn manual_split_and_merge_advance_through_operation_phases() { let gateway_url = Url::parse("http://127.0.0.1:18081/").unwrap(); @@ -887,15 +1021,17 @@ mod tests { metrics: Arc::new(HostMetrics::default()), operations: Arc::new(OperationRegistry::default()), }; - let response = split_partition( - State(state.clone()), - Path(("testdb".to_owned(), "testcoll".to_owned(), 0)), - Some(Json(SplitRequest { + let response = split_partition_inner( + state.clone(), + "testdb".to_owned(), + "testcoll".to_owned(), + 0, + SplitRequest { mode: SplitMode::Storage, epk: None, progression_mode: ProgressionMode::Manual, lock_duration_ms: None, - })), + }, ) .await .unwrap(); @@ -914,9 +1050,13 @@ mod tests { .child_partition_ids("testdb", "testcoll", &[0]) .is_empty()); - let succeeded = advance_operation(State(state.clone()), Path(operation_id.clone())) - .await - .unwrap(); + let completion_started = + advance_operation(State(state.clone()), Path(operation_id.clone())) + .await + .unwrap(); + assert_eq!(completion_started["phase"], "Swapping"); + drop(completion_started); + let succeeded = wait_for_phase(&state, &operation_id, "Succeeded").await; assert_eq!(succeeded["status"], "Succeeded"); assert_eq!(succeeded["phase"], "Succeeded"); assert_eq!(succeeded["children"].as_array().unwrap().len(), 2); @@ -927,31 +1067,22 @@ mod tests { .unwrap(); assert_eq!(queried["phase"], "Succeeded"); - let repeated = split_partition( - State(state.clone()), - Path(("testdb".to_owned(), "testcoll".to_owned(), 0)), - Some(Json(SplitRequest { + let operation_count = state.operations.records.lock().await.len(); + let repeated = split_partition_inner( + state.clone(), + "testdb".to_owned(), + "testcoll".to_owned(), + 0, + SplitRequest { mode: SplitMode::Epk, epk: Some(succeeded["splitEpk"].as_str().unwrap().to_owned()), progression_mode: ProgressionMode::Automatic, lock_duration_ms: None, - })), + }, ) - .await - .unwrap(); - let repeated_id = repeated.1["operationId"].as_str().unwrap().to_owned(); - let mut repeated_phase = None; - for _ in 0..100 { - let operation = get_operation(State(state.clone()), Path(repeated_id.clone())) - .await - .unwrap(); - repeated_phase = operation["phase"].as_str().map(str::to_owned); - if repeated_phase.as_deref() == Some("Failed") { - break; - } - tokio::task::yield_now().await; - } - assert_eq!(repeated_phase.as_deref(), Some("Failed")); + .await; + assert!(repeated.is_err()); + assert_eq!(state.operations.records.lock().await.len(), operation_count); let children = succeeded["children"] .as_array() @@ -975,25 +1106,64 @@ mod tests { .await .unwrap(); assert_eq!(merge_swapping["phase"], "Swapping"); - let merge_succeeded = advance_operation(State(state.clone()), Path(merge_id)) - .await - .unwrap(); + let merge_completion_started = + advance_operation(State(state.clone()), Path(merge_id.clone())) + .await + .unwrap(); + assert_eq!(merge_completion_started["phase"], "Swapping"); + drop(merge_completion_started); + let merge_succeeded = wait_for_phase(&state, &merge_id, "Succeeded").await; assert_eq!(merge_succeeded["phase"], "Succeeded"); assert_eq!(merge_succeeded["merged"].as_array().unwrap().len(), 2); - let invalid_duration = split_partition( - State(state), - Path(("testdb".to_owned(), "testcoll".to_owned(), 3)), - Some(Json(SplitRequest { + let invalid_duration = split_partition_inner( + state.clone(), + "testdb".to_owned(), + "testcoll".to_owned(), + 3, + SplitRequest { mode: SplitMode::Midpoint, epk: None, progression_mode: ProgressionMode::Manual, lock_duration_ms: Some(0), - })), + }, ) .await .unwrap_err(); assert_eq!(invalid_duration.status, StatusCode::BAD_REQUEST); + + let operation_count = state.operations.records.lock().await.len(); + let missing_partition = split_partition_inner( + state.clone(), + "testdb".to_owned(), + "testcoll".to_owned(), + 999, + SplitRequest { + mode: SplitMode::Epk, + epk: Some("01".to_owned()), + progression_mode: ProgressionMode::Automatic, + lock_duration_ms: None, + }, + ) + .await; + assert!(missing_partition.is_err()); + assert_eq!(state.operations.records.lock().await.len(), operation_count); + + let boundary = split_partition_inner( + state.clone(), + "testdb".to_owned(), + "testcoll".to_owned(), + children[0], + SplitRequest { + mode: SplitMode::Epk, + epk: Some("".to_owned()), + progression_mode: ProgressionMode::Automatic, + lock_duration_ms: None, + }, + ) + .await; + assert!(boundary.is_err()); + assert_eq!(state.operations.records.lock().await.len(), operation_count); } #[test] @@ -1001,4 +1171,107 @@ mod tests { assert!(parse_epk("ABC").is_err()); assert!(parse_epk("not-hex").is_err()); } + + #[test] + #[should_panic(expected = "conflicts with the response envelope")] + fn operation_result_rejects_reserved_fields() { + let mut operation = OperationRecord::running(None); + operation.succeed(json!({ "status": "not-the-envelope-status" })); + operation_response("op-test-1", &operation); + } + + #[tokio::test] + async fn optional_json_distinguishes_empty_and_rejected_bodies() { + let empty = axum::http::Request::builder() + .body(axum::body::Body::empty()) + .unwrap(); + let parsed: SplitRequest = parse_optional_json(empty).await.unwrap(); + assert!(matches!(parsed.mode, SplitMode::Midpoint)); + + let missing_content_type = axum::http::Request::builder() + .body(axum::body::Body::from(r#"{"mode":"epk","epk":"01"}"#)) + .unwrap(); + let error = parse_optional_json::(missing_content_type) + .await + .unwrap_err(); + assert_eq!(error.status, StatusCode::UNSUPPORTED_MEDIA_TYPE); + + for body in ["{", r#"{"unknown":true}"#] { + let request = axum::http::Request::builder() + .header(axum::http::header::CONTENT_TYPE, "application/json") + .body(axum::body::Body::from(body)) + .unwrap(); + let error = parse_optional_json::(request) + .await + .unwrap_err(); + assert_eq!(error.status, StatusCode::BAD_REQUEST); + } + } + + #[tokio::test] + async fn split_lro_round_trips_over_http() { + let gateway_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let account = + VirtualAccountConfig::new(vec![VirtualRegion::new("East US", gateway_url)]).unwrap(); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(account)); + emulator.store().create_database("testdb"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + emulator.store().create_container_with_config( + "testdb", + "testcoll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = router( + emulator, + "test-account".to_owned(), + Vec::new(), + Arc::new(HostMetrics::default()), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let (status, created) = http_json( + address, + "POST", + "/databases/testdb/containers/testcoll/partitions/0/split", + Some(r#"{"mode":"midpoint","progressionMode":"manual"}"#), + ) + .await; + assert_eq!(status, 202); + assert_eq!(created["phase"], "Preparing"); + let operation_id = created["operationId"].as_str().unwrap(); + + let operation_path = format!("/operations/{operation_id}"); + let advance_path = format!("{operation_path}/advance"); + let (status, swapping) = http_json(address, "POST", &advance_path, None).await; + assert_eq!(status, 200); + assert_eq!(swapping["phase"], "Swapping"); + let (status, still_swapping) = http_json(address, "POST", &advance_path, None).await; + assert_eq!(status, 200); + assert_eq!(still_swapping["phase"], "Swapping"); + + let mut terminal = None; + for _ in 0..100 { + let (status, operation) = http_json(address, "GET", &operation_path, None).await; + assert_eq!(status, 200); + if operation["phase"] == "Succeeded" { + terminal = Some(operation); + break; + } + tokio::task::yield_now().await; + } + let terminal = terminal.expect("operation did not reach Succeeded"); + assert_eq!(terminal["children"].as_array().unwrap().len(), 2); + assert!(terminal["splitEpk"].is_string()); + + server.abort(); + } } diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index 3f436706162..5307b1303ef 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -66,21 +66,16 @@ extends: Path: sdk/cosmos/live-platform-matrix.json Selection: sparse GenerateVMJobs: true - # Non-blocking vnext (Linux) emulator coverage. Adds a single extra job to - # the Build stage's matrix that runs the cosmos integration test suite - # against the local vnext Cosmos emulator container. The matrix entry - # carries `ContinueOnError: "true"` so failures here surface as - # "succeeded with issues" rather than blocking Release. Container - # lifecycle is owned by sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 - # (started when AZURE_COSMOS_EMULATOR_FLAVOR=vnext is set on the leg). - AdditionalMatrixConfigs: + # Emulator jobs run only in Build. They do not enter live stages, deploy + # Azure resources, or replace live-test credentials with local endpoints. + BuildOnlyAdditionalMatrixConfigs: - Name: Cosmos_vnext_emulator Path: sdk/cosmos/vnext-emulator-matrix.json Selection: all GenerateVMJobs: true - Name: Cosmos_inmemory_emulator - # Both jobs use port-zero host configs. Test setup parses the host's - # ready record and exports its OS-assigned account and management URLs. + # Both blocking jobs use port-zero host configs. Test setup parses the + # ready record and exports the OS-assigned account and management URLs. Path: sdk/cosmos/inmemory-emulator-matrix.json Selection: all GenerateVMJobs: true diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 index 5ae98f3fefd..5f1e93fb073 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 @@ -8,19 +8,6 @@ $ShutdownTimeout = 30 if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { - if ($env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 -eq 'true') { - try { - $healthUrl = ([System.Uri]::new([System.Uri]$env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT, 'health')).AbsoluteUri - $health = Invoke-RestMethod -Uri $healthUrl -TimeoutSec 5 -ErrorAction Stop - if ($health.connectivityProbes -lt 1 -or $health.gateway20Requests -lt 1) { - throw "Gateway 2.0 CI leg did not exercise the Gateway 2.0 path (probes=$($health.connectivityProbes), requests=$($health.gateway20Requests))." - } - Write-Host "Hosted Gateway 2.0 traffic verified (probes=$($health.connectivityProbes), requests=$($health.gateway20Requests))." - } catch { - LogError "Failed to verify hosted Gateway 2.0 traffic: $($_.Exception.Message)" - } - } - if ($env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) { $hostProcess = Get-Process -Id ([int]$env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) -ErrorAction SilentlyContinue } else { @@ -104,7 +91,6 @@ if ($env:AZURE_COSMOS_CONNECTION_STRING -eq "emulator" -or $env:AZURE_COSMOS_TEST_MODE = $null $env:AZURE_COSMOS_EMULATOR_HOST = $null $env:AZURE_COSMOS_INMEMORY_EMULATOR_PID = $null -$env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $null $env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT = $null $env:AZURE_COSMOS_INMEMORY_ACCOUNT_ENDPOINT = $null # Remove any --cfg=test_category="..." flag added by Test-Setup.ps1 or COSMOS_RUSTFLAGS. diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 3fbb76d560d..4c21b0f8671 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -44,9 +44,10 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { } if (-not $ready) { - LogGroupStart "Building hosted Cosmos DB in-memory emulator" + LogGroupStart "Testing and building hosted Cosmos DB in-memory emulator" Push-Location $repoRoot try { + Invoke-LoggedCommand 'cargo test -p azure_data_cosmos_emulator --all-features' Invoke-LoggedCommand 'cargo build -p azure_data_cosmos_emulator' } finally { Pop-Location @@ -71,7 +72,6 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { -RedirectStandardError $stderr ` -PassThru $env:AZURE_COSMOS_INMEMORY_EMULATOR_PID = $process.Id.ToString() - $env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $expectedGateway20.ToString().ToLowerInvariant() Write-Host "Started hosted emulator process $($process.Id) using '$configuration'." $deadline = (Get-Date).AddSeconds(60) @@ -123,8 +123,6 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { throw 'Hosted Cosmos DB in-memory emulator did not become ready within 60 seconds.' } LogGroupEnd - } else { - $env:AZURE_COSMOS_INMEMORY_EXPECT_GATEWAY20 = $expectedGateway20.ToString().ToLowerInvariant() } $env:AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT = $managementEndpoint @@ -134,6 +132,9 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $env:AZURE_COSMOS_TEST_MODE = 'required' $env:RUSTFLAGS = $env:RUSTFLAGS -replace '\s*--cfg=test_category="[^"]*"', '' $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator_inmemory`"" + if ($expectedGateway20) { + $env:RUSTFLAGS = "$($env:RUSTFLAGS) --cfg=test_category=`"emulator_inmemory_gateway_v2`"" + } $env:RUST_TEST_THREADS = '1' Write-Host "Hosted emulator is ready; RUSTFLAGS set to: $env:RUSTFLAGS" return diff --git a/sdk/cosmos/inmemory-emulator-matrix.json b/sdk/cosmos/inmemory-emulator-matrix.json index bde1ad622df..0eb56c7779d 100644 --- a/sdk/cosmos/inmemory-emulator-matrix.json +++ b/sdk/cosmos/inmemory-emulator-matrix.json @@ -11,7 +11,6 @@ } }, "RustToolchainName": ["stable"], - "AZURE_COSMOS_EMULATOR_FLAVOR": ["inmemory-v1", "inmemory-v2"], - "ContinueOnError": ["true"] + "AZURE_COSMOS_EMULATOR_FLAVOR": ["inmemory-v1", "inmemory-v2"] } } diff --git a/sdk/cosmos/vnext-emulator-matrix.json b/sdk/cosmos/vnext-emulator-matrix.json index 64be29f75b0..13e5bd59857 100644 --- a/sdk/cosmos/vnext-emulator-matrix.json +++ b/sdk/cosmos/vnext-emulator-matrix.json @@ -10,7 +10,6 @@ } }, "RustToolchainName": ["stable"], - "AZURE_COSMOS_EMULATOR_FLAVOR": ["vnext"], - "ContinueOnError": ["true"] + "AZURE_COSMOS_EMULATOR_FLAVOR": ["vnext"] } } From cabc4aa9a800a59015feab559733d0cd3f1457b1 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 10:22:53 +0000 Subject: [PATCH 32/44] Reacting to offline code review comments --- .../docs/in-memory-emulator-spec.md | 83 +-- .../driver/transport/gateway_v2_dispatch.rs | 35 +- .../src/in_memory_emulator/gateway_v2.rs | 135 +++- .../src/in_memory_emulator/operations.rs | 1 + .../emulator_tests/hosted_emulator_ci.rs | 114 ++++ .../in_memory_emulator_tests/error_cases.rs | 41 ++ .../azure_data_cosmos_emulator/Cargo.toml | 1 - .../azure_data_cosmos_emulator/README.md | 82 +++ .../config/ci-gateway-v1.json | 2 +- .../config/ci-gateway-v2.json | 2 +- .../azure_data_cosmos_emulator/docs/plan.md | 4 +- .../azure_data_cosmos_emulator/src/config.rs | 174 ++++- .../src/data_plane.rs | 6 + .../src/management.rs | 631 +++++++++++------- .../eng/scripts/Invoke-CosmosTestCleanup.ps1 | 15 +- .../eng/scripts/Invoke-CosmosTestSetup.ps1 | 91 ++- sdk/cosmos/inmemory-emulator-matrix.json | 11 +- sdk/cosmos/vnext-emulator-matrix.json | 10 +- 18 files changed, 1081 insertions(+), 357 deletions(-) create mode 100644 sdk/cosmos/azure_data_cosmos_emulator/README.md diff --git a/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md b/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md index e1395dfafb9..088922d489b 100644 --- a/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md +++ b/sdk/cosmos/azure_data_cosmos/docs/in-memory-emulator-spec.md @@ -566,48 +566,48 @@ Each physical partition is independently serialized: ### URL Path Parsing -| Pattern | Resource Type | -| ---------------------------------- | ------------------------------------------------------------ | -| `/` | Account | -| `/dbs` | DatabaseFeed (create/read/query) | -| `/dbs/{db}` | Database (read/delete) | -| `/dbs/{db}/colls` | ContainerFeed (create/read/query) | -| `/dbs/{db}/colls/{coll}` | Container (read/delete) | -| `/dbs/{db}/colls/{coll}/pkranges` | PartitionKeyRangeFeed (read feed) | -| `/dbs/{db}/colls/{coll}/docs` | DocumentFeed (create/upsert/read/query/query-plan/batch) | -| `/dbs/{db}/colls/{coll}/docs/{id}` | Document (read/replace/delete) | -| `/offers` | OfferFeed (read/query) | -| `/offers/{id}` | Offer (read/replace) | +| Pattern | Resource Type | +| ---------------------------------- | -------------------------------------------------------- | +| `/` | Account | +| `/dbs` | DatabaseFeed (create/read/query) | +| `/dbs/{db}` | Database (read/delete) | +| `/dbs/{db}/colls` | ContainerFeed (create/read/query) | +| `/dbs/{db}/colls/{coll}` | Container (read/delete) | +| `/dbs/{db}/colls/{coll}/pkranges` | PartitionKeyRangeFeed (read feed) | +| `/dbs/{db}/colls/{coll}/docs` | DocumentFeed (create/upsert/read/query/query-plan/batch) | +| `/dbs/{db}/colls/{coll}/docs/{id}` | Document (read/replace/delete) | +| `/offers` | OfferFeed (read/query) | +| `/offers/{id}` | Offer (read/replace) | ### Operation Resolution -| HTTP Method | Path | Headers | Operation | -| ----------- | --------------------------- | -------------------------------------------- | ------------------ | -| `GET` | `/` | — | ReadAccount | -| `POST` | `/dbs` | — | CreateDatabase | -| `GET` | `/dbs` | — | ReadFeedDatabases | -| `POST` | `/dbs` | `x-ms-documentdb-isquery: True` | QueryDatabases | -| `GET` | `/dbs/{db}` | — | ReadDatabase | -| `DELETE` | `/dbs/{db}` | — | DeleteDatabase | -| `POST` | `/dbs/{db}/colls` | — | CreateContainer | -| `GET` | `/dbs/{db}/colls` | — | ReadFeedContainers | -| `POST` | `/dbs/{db}/colls` | `x-ms-documentdb-isquery: True` | QueryContainers | -| `GET` | `/dbs/{db}/colls/{coll}` | — | ReadContainer | -| `DELETE` | `/dbs/{db}/colls/{coll}` | — | DeleteContainer | -| `GET` | `.../colls/{coll}/pkranges` | — | ReadPKRanges | -| `POST` | `.../docs` | — | Create | -| `POST` | `.../docs` | `x-ms-documentdb-is-upsert: True` | Upsert | -| `GET` | `.../docs` | — | ReadFeedItems | -| `POST` | `.../docs` | `x-ms-documentdb-isquery: True` | QueryItems | -| `POST` | `.../docs` | `x-ms-cosmos-is-query-plan-request: True` | QueryPlan | -| `POST` | `.../docs` | `x-ms-cosmos-is-batch-request: True` | Batch | -| `GET` | `.../docs/{id}` | — | Read | -| `PUT` | `.../docs/{id}` | — | Replace | -| `DELETE` | `.../docs/{id}` | — | Delete | -| `GET` | `/offers` | — | ReadFeedOffers | -| `POST` | `/offers` | `x-ms-documentdb-isquery: True` | QueryOffers | -| `GET` | `/offers/{id}` | — | ReadOffer | -| `PUT` | `/offers/{id}` | — | ReplaceOffer | +| HTTP Method | Path | Headers | Operation | +| ----------- | --------------------------- | ----------------------------------------- | ------------------ | +| `GET` | `/` | — | ReadAccount | +| `POST` | `/dbs` | — | CreateDatabase | +| `GET` | `/dbs` | — | ReadFeedDatabases | +| `POST` | `/dbs` | `x-ms-documentdb-isquery: True` | QueryDatabases | +| `GET` | `/dbs/{db}` | — | ReadDatabase | +| `DELETE` | `/dbs/{db}` | — | DeleteDatabase | +| `POST` | `/dbs/{db}/colls` | — | CreateContainer | +| `GET` | `/dbs/{db}/colls` | — | ReadFeedContainers | +| `POST` | `/dbs/{db}/colls` | `x-ms-documentdb-isquery: True` | QueryContainers | +| `GET` | `/dbs/{db}/colls/{coll}` | — | ReadContainer | +| `DELETE` | `/dbs/{db}/colls/{coll}` | — | DeleteContainer | +| `GET` | `.../colls/{coll}/pkranges` | — | ReadPKRanges | +| `POST` | `.../docs` | — | Create | +| `POST` | `.../docs` | `x-ms-documentdb-is-upsert: True` | Upsert | +| `GET` | `.../docs` | — | ReadFeedItems | +| `POST` | `.../docs` | `x-ms-documentdb-isquery: True` | QueryItems | +| `POST` | `.../docs` | `x-ms-cosmos-is-query-plan-request: True` | QueryPlan | +| `POST` | `.../docs` | `x-ms-cosmos-is-batch-request: True` | Batch | +| `GET` | `.../docs/{id}` | — | Read | +| `PUT` | `.../docs/{id}` | — | Replace | +| `DELETE` | `.../docs/{id}` | — | Delete | +| `GET` | `/offers` | — | ReadFeedOffers | +| `POST` | `/offers` | `x-ms-documentdb-isquery: True` | QueryOffers | +| `GET` | `/offers/{id}` | — | ReadOffer | +| `PUT` | `/offers/{id}` | — | ReplaceOffer | ### Region Routing @@ -770,7 +770,10 @@ One-shot: next read to the physical partition containing the given partition key ## 10. ETag & Optimistic Concurrency - New quoted UUID ETag generated on every write. -- **If-Match**: Compared on Replace/Delete → 412 on mismatch. +- **If-Match**: Compared on Replace/Delete → 412 on mismatch. Also enforced on Upsert + when the target item already exists; an Upsert against a non-existent item ignores + `If-Match` and creates the item (mirrors real Cosmos DB — preconditions only apply + to items that already exist). - **If-None-Match**: Reserved for future use (create-if-not-exists pattern). - ETag returned in both `etag` header and `_etag` body property. diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index 2c0ace1be5f..a0055d5cb58 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -407,7 +407,26 @@ pub(crate) fn wrap_request_for_gateway_v2( pub(crate) fn unwrap_response_for_gateway_v2( response: HttpResponse, ) -> azure_core::Result { - let outer_headers = response.headers.clone(); + // Only these headers are ever consulted as an outer-HTTP fallback below, + // so capture just their (at most six) values instead of cloning the + // entire response header map up front. + let outer_fallbacks: Vec<(HeaderName, String)> = [ + response_header_names::SERVER_DURATION_MS, + response_header_names::LSN, + response_header_names::ITEM_LSN, + response_header_names::GLOBAL_COMMITTED_LSN, + response_header_names::QUERY_METRICS, + response_header_names::INDEX_METRICS, + ] + .into_iter() + .filter_map(|name| { + let header = HeaderName::from_static(name); + response + .headers + .get_optional_str(&header) + .map(|value| (header, value.to_owned())) + }) + .collect(); let response = RntbdResponse::read(&response.body)?; let status = u16::from(response.status.status_code()); if !(100..=599).contains(&status) { @@ -476,19 +495,9 @@ pub(crate) fn unwrap_response_for_gateway_v2( if let Some(owner_full_name) = response.owner_full_name { headers.insert(response_header_names::OWNER_FULL_NAME, owner_full_name); } - for name in [ - response_header_names::SERVER_DURATION_MS, - response_header_names::LSN, - response_header_names::ITEM_LSN, - response_header_names::GLOBAL_COMMITTED_LSN, - response_header_names::QUERY_METRICS, - response_header_names::INDEX_METRICS, - ] { - let header = HeaderName::from_static(name); + for (header, value) in outer_fallbacks { if headers.get_optional_str(&header).is_none() { - if let Some(value) = outer_headers.get_optional_str(&header) { - headers.insert(header, value.to_owned()); - } + headers.insert(header, value); } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs index 18c2866b04f..fc9b03cea3b 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs @@ -31,8 +31,17 @@ impl InMemoryEmulatorHttpClient { request: &Request, ) -> crate::error::Result { let request_body: Bytes = request.body().into(); - let frame = - RntbdRequestFrame::read(request_body.as_ref()).map_err(gateway_v2_bad_request)?; + // A frame that fails to parse has no usable `activityId` field, so + // there is nothing to echo back — but the response must still be a + // well-formed RNTBD frame (not a bare JSON body) so that any client + // speaking Gateway 2.0, not just the Rust driver, can decode the + // failure the same way it decodes every other error on this path. + let frame = match RntbdRequestFrame::read(request_body.as_ref()) { + Ok(frame) => frame, + Err(error) => { + return encode_error_response(gateway_v2_bad_request(error), Uuid::new_v4()).await + } + }; let activity_id = frame.activity_id; let request = match decode_request(request, frame, self.store().config().consistency()) { Ok(request) => request, @@ -706,6 +715,128 @@ mod tests { assert_eq!(framed.activity_id, activity_id); } + #[tokio::test] + async fn malformed_frame_bytes_are_framed_with_matching_status() { + let gateway_v2_url = Url::parse("http://127.0.0.1:18444/").unwrap(); + let region = VirtualRegion::new("East US", "http://127.0.0.1:18081/".parse().unwrap()) + .with_gateway_v2_url(gateway_v2_url.clone()); + let emulator = + InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![region]).unwrap()); + let mut request = Request::new( + gateway_v2_url.join("dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + // Not a valid RNTBD frame at all: too short to even contain a header + // length prefix, let alone resource/operation type and activity id. + // A generic h2c client that sends garbage bytes must still get back + // a response it can decode as RNTBD, not a bare JSON error body. + request.set_body(vec![0x01, 0x02, 0x03]); + + let response = emulator + .execute_gateway_v2_request(&request) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BadRequest); + let framed = RntbdResponse::read(response.body().as_ref()) + .expect("malformed-frame error response must itself be a well-formed RNTBD frame"); + assert_eq!(framed.status.status_code(), StatusCode::BadRequest); + } + + #[tokio::test] + async fn multi_region_gateway_v2_endpoints_each_serve_their_own_region() { + let east_gateway = Url::parse("http://127.0.0.1:18081/").unwrap(); + let east_gateway_v2 = Url::parse("http://127.0.0.1:18444/").unwrap(); + let west_gateway = Url::parse("http://127.0.0.1:18082/").unwrap(); + let west_gateway_v2 = Url::parse("http://127.0.0.1:18445/").unwrap(); + let east = VirtualRegion::new("East US", east_gateway.clone()) + .with_gateway_v2_url(east_gateway_v2.clone()); + let west = VirtualRegion::new("West US", west_gateway) + .with_gateway_v2_url(west_gateway_v2.clone()); + let emulator = + InMemoryEmulatorHttpClient::new(VirtualAccountConfig::new(vec![east, west]).unwrap()); + let store = emulator.store(); + store.create_database("db"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + store.create_container_with_config( + "db", + "coll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + + // Seed through the write region's standard gateway, then let the + // write replicate to the second region before reading it back + // through each region's own Gateway 2.0 endpoint. + let mut seed = Request::new( + east_gateway.join("dbs/db/colls/coll/docs").unwrap(), + Method::Post, + ); + seed.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + seed.set_body( + serde_json::to_vec(&serde_json::json!({ "id": "item1", "pk": "pk1", "value": 42 })) + .unwrap(), + ); + assert!(emulator + .execute_request(&seed) + .await + .unwrap() + .status() + .is_success()); + store.drain_pending_replications().await; + + for gateway_v2_url in [&east_gateway_v2, &west_gateway_v2] { + let activity_id = Uuid::new_v4(); + let frame = RntbdRequestFrame { + resource_type: ResourceType::Document, + operation_type: OperationType::Read, + activity_id, + metadata: vec![ + Token::database_name("db".to_owned()), + Token::collection_name("coll".to_owned()), + Token::document_name("item1".to_owned()), + Token::partition_key(r#"["pk1"]"#.to_owned()), + Token::payload_present(false), + ], + body: None, + }; + let mut bytes = Vec::new(); + frame.write(&mut bytes).unwrap(); + let mut request = Request::new( + gateway_v2_url.join("dbs/db/colls/coll/docs/item1").unwrap(), + Method::Post, + ); + request.set_body(bytes); + + let response = emulator + .execute_gateway_v2_request(&request) + .await + .unwrap() + .try_into_raw_response() + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::Ok, + "each region's Gateway 2.0 endpoint ({gateway_v2_url}) must independently serve its own replicated data" + ); + let framed = RntbdResponse::read(response.body().as_ref()).unwrap(); + assert_eq!(framed.status.status_code(), StatusCode::Ok); + assert_eq!(framed.activity_id, activity_id); + } + } + #[tokio::test] async fn throttled_response_uses_matching_outer_and_inner_status() { let mut headers = Headers::new(); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs index 09dd6730df4..82b912aee03 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs @@ -4587,6 +4587,7 @@ fn handle_read( } let lsn = partition.current_lsn(); + drop(docs); let headers = Some(PointResponseHeaders::from_partition( partition, store.next_transport_request_id(), diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs index 6707a96cb6f..fe014d05de3 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/hosted_emulator_ci.rs @@ -58,3 +58,117 @@ async fn configured_host_mode_is_exercised() -> Result<(), Box Result<(), Box> { + let management_endpoint = std::env::var("AZURE_COSMOS_INMEMORY_MANAGEMENT_ENDPOINT")?; + let client = reqwest::Client::new(); + + DriverTestClient::run_with_unique_db(async |context, database| { + let container_name = context.unique_container_name(); + let container = context + .create_container(&database, &container_name, "/pk") + .await?; + let db_name = database + .name() + .ok_or("database reference must be name-based")?; + + let created = context + .create_item( + &container, + "split-item-1", + "pk-a", + br#"{"id":"split-item-1","pk":"pk-a","value":1}"#, + ) + .await?; + context + .create_item( + &container, + "split-item-2", + "pk-z", + br#"{"id":"split-item-2","pk":"pk-z","value":2}"#, + ) + .await?; + + // The management API's partition IDs are opaque to normal Cosmos + // clients and never used for routing by production code — recover + // one here purely to target the split, from the diagnostic-only + // `x-ms-documentdb-partitionkeyrangeid` response header. + let partition_id: u32 = created + .headers() + .partition_key_range_id + .as_deref() + .and_then(|id| id.parse().ok()) + .ok_or("could not determine partition id from response headers")?; + + // Trigger a real split through the management API — the same real + // topology-mutation infrastructure a test author would drive + // interactively — and wait for it to complete. `midpoint` mode + // (unlike `storage`) never depends on how many documents happen to + // already be in the target partition, so it doesn't matter which of + // the container's default partitions `partition_id` landed on. + let split_url = format!( + "{management_endpoint}databases/{db_name}/containers/{container_name}/partitions/{partition_id}/split" + ); + let created_operation = client + .post(&split_url) + .header("content-type", "application/json") + .body(serde_json::to_vec(&serde_json::json!({ "mode": "midpoint" }))?) + .send() + .await? + .error_for_status()? + .bytes() + .await?; + let created_operation: serde_json::Value = serde_json::from_slice(&created_operation)?; + let operation_id = created_operation["operationId"] + .as_str() + .ok_or("split response missing operationId")?; + + let operation_url = format!("{management_endpoint}operations/{operation_id}"); + let mut terminal = None; + for _ in 0..100 { + let body = client + .get(&operation_url) + .send() + .await? + .error_for_status()? + .bytes() + .await?; + let operation: serde_json::Value = serde_json::from_slice(&body)?; + if operation["phase"] == "Succeeded" || operation["phase"] == "Failed" { + terminal = Some(operation); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + let terminal = terminal.ok_or("split operation did not reach a terminal phase")?; + assert_eq!( + terminal["phase"], "Succeeded", + "split must succeed: {terminal:?}" + ); + + // The production driver has no idea a split just happened. Reading + // both items back must transparently refresh its stale PKRange + // cache and route to the new child partitions. + let read_back_1 = context.read_item(&container, "split-item-1", "pk-a").await?; + assert!(read_back_1.status().is_success()); + let read_back_2 = context.read_item(&container, "split-item-2", "pk-z").await?; + assert!(read_back_2.status().is_success()); + + Ok(()) + }) + .await +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs index 591f2907207..9927da5ff39 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/error_cases.rs @@ -68,6 +68,47 @@ async fn upsert_missing_with_if_match_creates_item() { assert_eq!(response.status(), StatusCode::Created); } +#[tokio::test] +async fn upsert_existing_with_mismatched_if_match_returns_412() { + let ctx = setup_single_region().await; + let body = serde_json::json!({"id": "existing", "pk": "pk1", "value": 1}); + let request = create_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + true, + ); + let response = ctx.emulator.execute_request(&request).await.unwrap(); + let (status, _headers, created) = collect_response(response).await; + assert_eq!(status, StatusCode::Created); + let etag = created["_etag"].as_str().unwrap(); + assert_ne!( + etag, "\"stale\"", + "test fixture assumption: real etags are never literally 'stale'" + ); + + let replacement = serde_json::json!({"id": "existing", "pk": "pk1", "value": 2}); + let mut request = create_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &replacement, + r#"["pk1"]"#, + false, + ); + request + .headers_mut() + .insert(IS_UPSERT.clone(), HeaderValue::from_static("True")); + request + .headers_mut() + .insert(IF_MATCH.clone(), HeaderValue::from_static("\"stale\"")); + + let response = ctx.emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), StatusCode::PreconditionFailed); +} + #[tokio::test] async fn replace_nonexistent_404() { let ctx = setup_single_region().await; diff --git a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml index 65156bee51f..03c33ce6f0c 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml @@ -24,7 +24,6 @@ tokio = { workspace = true, features = [ "io-util", "net", "rt-multi-thread", - "signal", "sync", ] } tracing.workspace = true diff --git a/sdk/cosmos/azure_data_cosmos_emulator/README.md b/sdk/cosmos/azure_data_cosmos_emulator/README.md new file mode 100644 index 00000000000..e45d31f27fe --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_emulator/README.md @@ -0,0 +1,82 @@ +# azure_data_cosmos_emulator + +Out-of-process host for the Azure Cosmos DB Rust SDK's in-memory test emulator +(`azure_data_cosmos_driver::in_memory_emulator`). It exposes the same +deterministic, memory-backed emulator used by the driver's own Rust unit +tests over real network ports, so any client — not just Rust — can exercise +it over the Cosmos DB wire protocol. + +This is an **SDK engineering and test tool**, not a supported customer +product: it provides no service compatibility, durability, performance, or +support guarantees. See +[`docs/adr/001_build_memory_backed_sdk_test_emulator.md`](docs/adr/001_build_memory_backed_sdk_test_emulator.md) +for the full rationale and scope, and [`docs/plan.md`](docs/plan.md) for the +complete design (configuration schema, management REST API, Gateway 2.0 +support, and CI integration). + +`publish = false` — this crate is never published to crates.io. + +## What it hosts + +- **Gateway V1** (JSON REST) on every configured region — always on. +- **Gateway 2.0** (RNTBD over cleartext HTTP/2) per region, when the region's + config sets `gateway20Port`. +- A **management REST API** for emulator-only control-plane actions with no + Cosmos gateway equivalent: partition split/merge (as long-running + operations), per-partition-failover toggling, and replication pause/resume. + +## Quick start + +Build and run against one of the sample configs checked into +[`config/`](config/): + +```sh +cargo build -p azure_data_cosmos_emulator +./target/debug/azure_data_cosmos_emulator --config sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json +``` + +The host writes one JSON `ready` record to stdout once every listener is +bound: + +```json +{ + "event": "ready", + "managementEndpoint": "http://127.0.0.1:49150/", + "accountEndpoint": "http://127.0.0.1:49151/", + "regions": [ + { "name": "East US", "gatewayEndpoint": "http://127.0.0.1:49151/" } + ] +} +``` + +Logs go to stderr, so automation can parse stdout without filtering. Use the +resolved `accountEndpoint` as a normal Cosmos DB connection string endpoint, +and the `managementEndpoint` to drive control-plane actions, e.g.: + +```sh +curl -X POST "http://127.0.0.1:49150/databases/testdb/containers/testcoll/partitions/0/split" +``` + +## Configuration + +A single JSON file (`--config`) describes the account topology, the +databases/containers to create, and optional seed items — all applied on +startup. See [`docs/plan.md`](docs/plan.md#4-configuration-file) for the full +field reference; [`config/ci-gateway-v1.json`](config/ci-gateway-v1.json) and +[`config/ci-gateway-v2.json`](config/ci-gateway-v2.json) are minimal, +CI-oriented examples (both use port `0` for every listener, so the OS assigns +free ports and the resolved endpoints come from the `ready` record). + +## Testing + +```sh +cargo test -p azure_data_cosmos_emulator --all-features +``` + +The hosted emulator is also exercised end-to-end by +`azure_data_cosmos_driver`'s and `azure_data_cosmos`'s existing emulator test +suites, gated behind `test_category = "emulator_inmemory"` (and +`"emulator_inmemory_gateway_v2"` for Gateway 2.0-specific tests). See +[`sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1`](../eng/scripts/Invoke-CosmosTestSetup.ps1) +for how CI builds, starts, and health-checks the host process before running +those suites. diff --git a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json index 17bfc7e8c1c..2d3cd6a1468 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json +++ b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v1.json @@ -2,7 +2,7 @@ "account": { "id": "emulator-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "perPartitionFailover": false, "throttling": false, "regions": [ diff --git a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json index 86263f36547..3ddce19fda2 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json +++ b/sdk/cosmos/azure_data_cosmos_emulator/config/ci-gateway-v2.json @@ -2,7 +2,7 @@ "account": { "id": "emulator-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "perPartitionFailover": false, "throttling": false, "regions": [ diff --git a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md index e8864871373..02c64aab611 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/docs/plan.md @@ -151,7 +151,7 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-006. "account": { "id": "emulator-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "perPartitionFailover": false, "throttling": false, "regions": [ @@ -189,7 +189,7 @@ API can further modify state at runtime. (YAML support is deferred; see ADR-006. | Path | Type | Notes | | ----------------------------------------- | --------------------- | -------------------------------------------------------------------------------------------------- | | `account.writeMode` | `"single" \| "multi"` | Maps to `WriteMode`. In `single`, the first region is the hub/write region. | -| `account.consistency` | enum | `Strong \| BoundedStaleness \| Session \| ConsistentPrefix \| Eventual`. | +| `account.consistency` | enum | `strong \| boundedStaleness \| session \| consistentPrefix \| eventual` (camelCase, matching `writeMode`). | | `account.perPartitionFailover` | bool | Initial `enablePerPartitionFailoverBehavior`; can be toggled at runtime. | | `account.throttling` | bool | Enables per-partition RU/s enforcement (429/3200). | | `account.regions[].gatewayPort` | u16, optional | Standard gateway port. Missing or `0` requests an OS-assigned port. | diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs index 4b927df331e..7420fb9238e 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/config.rs @@ -121,13 +121,14 @@ struct ReplicationOverride { } #[derive(Clone, Copy, Debug, Deserialize)] -#[serde(rename_all = "lowercase")] +#[serde(rename_all = "camelCase")] enum WriteModeConfig { Single, Multi, } #[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] enum ConsistencyConfig { Strong, BoundedStaleness, @@ -178,8 +179,15 @@ enum ListenerSlot { impl EmulatorConfig { pub(crate) async fn load(path: &Path) -> Result { - let contents = tokio::fs::read(path).await?; - let config: Self = serde_json::from_slice(&contents)?; + let contents = tokio::fs::read(path) + .await + .map_err(|error| format!("failed to read config file '{}': {error}", path.display()))?; + let config: Self = serde_json::from_slice(&contents).map_err(|error| { + format!( + "failed to parse config file '{}' as JSON: {error}", + path.display() + ) + })?; config.validate()?; Ok(config) } @@ -306,16 +314,11 @@ impl EmulatorConfig { for database in &self.databases { store.create_database(&database.id); for container in &database.containers { - let mut container_config = - ContainerConfig::new().with_partition_count(container.partition_count); - if let Some(throughput) = container.throughput { - container_config = container_config.with_throughput(throughput); - } store.create_container_with_config( &database.id, &container.id, container.partition_key.clone(), - container_config.build()?, + build_container_config(container)?, ); for seed_item in &container.seed_items { @@ -391,18 +394,29 @@ impl EmulatorConfig { ) .into()); } - let mut container_config = - ContainerConfig::new().with_partition_count(container.partition_count); - if let Some(throughput) = container.throughput { - container_config = container_config.with_throughput(throughput); - } - container_config.build()?; + build_container_config(container)?; } } Ok(()) } } +/// Builds the driver-level [`ContainerConfig`] for a configured container. +/// +/// Shared by [`EmulatorConfig::validate`] (which discards the built config, +/// using it only to surface a validation error before anything is +/// provisioned) and [`EmulatorConfig::provision`] (which uses it to actually +/// create the container), so a future container option added to one path +/// cannot silently be forgotten on the other. +fn build_container_config(container: &ContainerSettings) -> Result { + let mut container_config = + ContainerConfig::new().with_partition_count(container.partition_count); + if let Some(throughput) = container.throughput { + container_config = container_config.with_throughput(throughput); + } + Ok(container_config.build()?) +} + fn validate_resource_id(resource_type: &str, id: &str) -> Result<()> { if id.trim().is_empty() { return Err(format!("{resource_type} IDs must not be empty").into()); @@ -512,7 +526,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US", "gatewayPort": 0 }] }, "management": { "port": 0 }, @@ -562,7 +576,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US", "gatewayPort": 9090 }] }, "management": { "port": 9090 } @@ -578,7 +592,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US", "gateway20Port": 0 }] } })) @@ -624,7 +638,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [ { "name": "East US", "gatewayPort": 18081, "regionId": 1 }, { "name": "east us", "gatewayPort": 18082, "regionId": 1 } @@ -641,7 +655,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US" }] }, "databases": [{ "id": "db" }, { "id": "db" }] @@ -653,7 +667,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US" }] }, "databases": [{ @@ -674,7 +688,7 @@ mod tests { "account": { "id": "test-account", "writeMode": "single", - "consistency": "Session", + "consistency": "session", "regions": [{ "name": "East US" }] }, "databases": [ @@ -705,4 +719,120 @@ mod tests { let response = emulator.execute_request(&request).await.unwrap(); assert_eq!(response.status(), azure_core::http::StatusCode::NotFound); } + + #[tokio::test] + async fn applies_replication_overrides() { + let config: EmulatorConfig = serde_json::from_value(serde_json::json!({ + "account": { + "id": "test-account", + "writeMode": "single", + "consistency": "session", + "regions": [ + { "name": "East US", "gatewayPort": 0 }, + { "name": "West US", "gatewayPort": 0 } + ], + "replication": { "minDelayMs": 0, "maxDelayMs": 0 }, + "replicationOverrides": [ + { "source": "East US", "target": "West US", "minDelayMs": 5000, "maxDelayMs": 5000 } + ] + } + })) + .unwrap(); + config.validate().unwrap(); + let bound_host = config.bind().await.unwrap(); + let bindings: Vec<_> = bound_host + .gateways + .iter() + .map(|gateway| gateway.binding()) + .collect(); + let east_url = bindings[0].gateway_url.clone(); + let west_url = bindings[1].gateway_url.clone(); + let emulator = config.create_emulator(&bindings).unwrap(); + emulator.store().create_database("testdb"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + emulator.store().create_container_with_config( + "testdb", + "testcoll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + + let mut request = Request::new( + east_url.join("dbs/testdb/colls/testcoll/docs").unwrap(), + Method::Post, + ); + request.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + request.set_body( + serde_json::to_vec(&serde_json::json!({ "id": "item1", "pk": "pk1" })).unwrap(), + ); + let response = emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), azure_core::http::StatusCode::Created); + + // The account-wide default (0ms) would have replicated almost + // instantly; the override for East US -> West US specifically is + // 5s, so after a much shorter wait West US must not see the write + // yet. This is only a meaningful assertion if the per-pair override + // — not the account default — is actually the one being applied. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let mut read = Request::new( + west_url + .join("dbs/testdb/colls/testcoll/docs/item1") + .unwrap(), + Method::Get, + ); + read.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + let response = emulator.execute_request(&read).await.unwrap(); + assert_eq!( + response.status(), + azure_core::http::StatusCode::NotFound, + "replicationOverrides' longer delay must be applied instead of the account default" + ); + } + + #[tokio::test] + async fn load_reports_a_clean_error_naming_a_missing_config_file() { + let missing_path = + Path::new("azure-data-cosmos-emulator-this-config-file-does-not-exist.json"); + let error = EmulatorConfig::load(missing_path) + .await + .expect_err("loading a nonexistent path must return an error, not panic"); + assert!( + error + .to_string() + .contains("azure-data-cosmos-emulator-this-config-file-does-not-exist.json"), + "error must name the offending path, got: {error}" + ); + } + + #[tokio::test] + async fn load_reports_a_clean_error_for_malformed_json() { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("azure-data-cosmos-emulator-test-{unique}.json")); + tokio::fs::write(&path, b"{ not valid json").await.unwrap(); + + let error = EmulatorConfig::load(&path).await; + tokio::fs::remove_file(&path).await.ok(); + + let error = error.expect_err("malformed JSON must return an error, not panic"); + assert!( + error.to_string().contains(&path.display().to_string()), + "error must name the offending path, got: {error}" + ); + } } diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs index 481b4e9b29a..a2da0ca2773 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/data_plane.rs @@ -95,6 +95,12 @@ pub(crate) async fn into_cosmos_request( .map_err(|error| (StatusCode::PAYLOAD_TOO_LARGE, error.to_string()))?; let mut cosmos_request = CosmosRequest::new(url, method); + // `azure_core::http::headers::Headers` is backed by a map keyed on header + // name, so a repeated header name here collapses to its last value — + // matching `into_http_response`'s output below, which can only ever + // iterate a `Headers` map and therefore can never observe (or forward) + // more than one value per name either. Both directions of this bridge + // are consistently single-valued-per-header-name. for (name, value) in &parts.headers { let value = value .to_str() diff --git a/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs index dccac8f27d5..63c3ce93210 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs +++ b/sdk/cosmos/azure_data_cosmos_emulator/src/management.rs @@ -15,8 +15,8 @@ use std::{ use axum::{ body::to_bytes, - extract::{Path, Request, State}, - http::StatusCode, + extract::{FromRequest, FromRequestParts, Path, Request, State}, + http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, routing::{get, post, put}, Json, Router, @@ -24,7 +24,7 @@ use axum::{ use azure_data_cosmos_driver::in_memory_emulator::{ EmulatorStore, Epk, InMemoryEmulatorHttpClient, ManualControlPlaneOperation, WriteMode, }; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::json; use tokio::{net::TcpListener, sync::Mutex}; @@ -32,6 +32,11 @@ use crate::{config::GatewayBinding, metrics::HostMetrics}; const MAX_CONTROL_PLANE_LOCK_DURATION_MS: u64 = 60_000; const MAX_MANAGEMENT_BODY_SIZE: usize = 1024 * 1024; +// Kept deliberately simple: a full scan-and-retain over a HashMap this small +// is cheap, and 1_000 concurrently-tracked operations (most short-lived CI +// runs will never approach this) is already a generous ceiling before we +// bother reclaiming memory from long-terminal split/merge records. +const OPERATION_EVICTION_THRESHOLD: usize = 1_000; #[derive(Clone)] struct ManagementState { @@ -192,33 +197,85 @@ struct OperationRecord { action: Option, } -enum OperationAction { - PendingSplit { - database: String, - container: String, - parent: u32, - mode: SplitMode, - split_epk: Epk, - }, - ActiveSplit { +/// The operation-specific data for a long-running control-plane operation. +/// +/// Deliberately separate from [`OperationAction`]'s pending/active state so +/// that a future operation kind (e.g. PR2's region offline/online or +/// write-region failover) only needs a new variant here — the +/// pending/active/advance/registry machinery around it does not change. +enum OperationKind { + Split { database: String, container: String, parent: u32, mode: SplitMode, split_epk: Epk, - operation: ManualControlPlaneOperation, }, - PendingMerge { + Merge { database: String, container: String, partitions: [u32; 2], }, - ActiveMerge { - database: String, - container: String, - partitions: [u32; 2], +} + +impl OperationKind { + /// Locks the operation's target partition(s) and returns the handle that + /// completes it once released. + fn begin_manual(&self, store: &Arc) -> ManualControlPlaneOperation { + match self { + Self::Split { + database, + container, + parent, + split_epk, + .. + } => { + store.begin_manual_split_partition(database, container, *parent, split_epk.clone()) + } + Self::Merge { + database, + container, + partitions, + } => store.begin_manual_merge_partitions( + database, + container, + partitions[0], + partitions[1], + ), + } + } + + /// Releases the manual operation's lock and reports the operation-kind-specific result. + async fn complete( + self, + store: &Arc, operation: ManualControlPlaneOperation, - }, + ) -> Result { + match self { + Self::Split { + database, + container, + parent, + mode, + split_epk, + } => { + complete_split( + store, database, container, parent, mode, split_epk, operation, + ) + .await + } + Self::Merge { + database, + container, + partitions, + } => complete_merge(store, database, container, partitions, operation).await, + } + } +} + +enum OperationAction { + Pending(OperationKind), + Active(OperationKind, ManualControlPlaneOperation), } impl OperationRegistry { @@ -228,6 +285,22 @@ impl OperationRegistry { } } +/// Evicts terminal (`Succeeded`/`Failed`) operations once the registry grows +/// past [`OPERATION_EVICTION_THRESHOLD`], so a long-lived host process +/// doesn't retain every split/merge record forever. Must be called while +/// holding the `records` lock. +fn evict_terminal_operations_if_needed(records: &mut HashMap) { + if records.len() < OPERATION_EVICTION_THRESHOLD { + return; + } + records.retain(|_, record| { + !matches!( + record.status, + OperationStatus::Succeeded | OperationStatus::Failed + ) + }); +} + impl OperationRecord { fn running(action: Option) -> Self { Self { @@ -311,17 +384,9 @@ struct SplitResponse { split_epk: String, } -struct SplitOperationDetails { - database: String, - container: String, - parent: u32, - mode: SplitMode, - split_epk: Epk, -} - async fn split_partition( State(state): State, - Path((database, container, partition_id)): Path<(String, String, u32)>, + ApiPath((database, container, partition_id)): ApiPath<(String, String, u32)>, request: Request, ) -> ApiResult<(StatusCode, Json)> { let request = parse_optional_json(request).await?; @@ -353,28 +418,27 @@ async fn split_partition_inner( let operation_id = state.operations.next_id("split"); let action = match request.progression_mode { ProgressionMode::Automatic => None, - ProgressionMode::Manual => Some(OperationAction::PendingSplit { + ProgressionMode::Manual => Some(OperationAction::Pending(OperationKind::Split { database: database.clone(), container: container.clone(), parent: partition_id, mode: request.mode, split_epk: split_epk.clone(), - }), + })), }; let operation = OperationRecord::running(action); let response = operation_response(&operation_id, &operation); - state - .operations - .records - .lock() - .await - .insert(operation_id.clone(), operation); + { + let mut records = state.operations.records.lock().await; + records.insert(operation_id.clone(), operation); + evict_terminal_operations_if_needed(&mut records); + } if request.progression_mode == ProgressionMode::Automatic { - spawn_automatic_split( + spawn_automatic( state, operation_id, - SplitOperationDetails { + OperationKind::Split { database, container, parent: partition_id, @@ -440,35 +504,36 @@ struct MergeResponse { async fn merge_partitions( State(state): State, - Path((database, container)): Path<(String, String)>, - Json(request): Json, + ApiPath((database, container)): ApiPath<(String, String)>, + ApiJson(request): ApiJson, ) -> ApiResult<(StatusCode, Json)> { let lock_duration = validate_progression(request.progression_mode, request.lock_duration_ms)?; let operation_id = state.operations.next_id("merge"); let action = match request.progression_mode { ProgressionMode::Automatic => None, - ProgressionMode::Manual => Some(OperationAction::PendingMerge { + ProgressionMode::Manual => Some(OperationAction::Pending(OperationKind::Merge { database: database.clone(), container: container.clone(), partitions: request.partition_ids, - }), + })), }; let operation = OperationRecord::running(action); let response = operation_response(&operation_id, &operation); - state - .operations - .records - .lock() - .await - .insert(operation_id.clone(), operation); + { + let mut records = state.operations.records.lock().await; + records.insert(operation_id.clone(), operation); + evict_terminal_operations_if_needed(&mut records); + } if request.progression_mode == ProgressionMode::Automatic { - spawn_automatic_merge( + spawn_automatic( state, operation_id, - database, - container, - request.partition_ids, + OperationKind::Merge { + database, + container, + partitions: request.partition_ids, + }, lock_duration, ); } @@ -476,64 +541,19 @@ async fn merge_partitions( Ok((StatusCode::ACCEPTED, Json(response))) } -fn spawn_automatic_split( - state: ManagementState, - operation_id: String, - details: SplitOperationDetails, - lock_duration: Duration, -) { - tokio::spawn(async move { - let SplitOperationDetails { - database, - container, - parent, - mode, - split_epk, - } = details; - let store = state.emulator.store(); - let operation = { - let mut operations = state.operations.records.lock().await; - let operation = store.begin_manual_split_partition( - &database, - &container, - parent, - split_epk.clone(), - ); - if let Some(record) = operations.get_mut(&operation_id) { - record.phase = OperationPhase::Swapping; - } - operation - }; - if !lock_duration.is_zero() { - tokio::time::sleep(lock_duration).await; - } - let mut operations = state.operations.records.lock().await; - let outcome = complete_split( - &store, database, container, parent, mode, split_epk, operation, - ) - .await; - finish_operation_locked(&mut operations, &operation_id, outcome); - }); -} - -fn spawn_automatic_merge( +/// Spawns the background task that advances an `automatic`-progression +/// operation from `Swapping` through to its terminal phase. +fn spawn_automatic( state: ManagementState, operation_id: String, - database: String, - container: String, - partitions: [u32; 2], + kind: OperationKind, lock_duration: Duration, ) { tokio::spawn(async move { let store = state.emulator.store(); let operation = { let mut operations = state.operations.records.lock().await; - let operation = store.begin_manual_merge_partitions( - &database, - &container, - partitions[0], - partitions[1], - ); + let operation = kind.begin_manual(&store); if let Some(record) = operations.get_mut(&operation_id) { record.phase = OperationPhase::Swapping; } @@ -543,7 +563,7 @@ fn spawn_automatic_merge( tokio::time::sleep(lock_duration).await; } let mut operations = state.operations.records.lock().await; - let outcome = complete_merge(&store, database, container, partitions, operation).await; + let outcome = kind.complete(&store, operation).await; finish_operation_locked(&mut operations, &operation_id, outcome); }); } @@ -616,7 +636,7 @@ async fn complete_merge( async fn get_operation( State(state): State, - Path(operation_id): Path, + ApiPath(operation_id): ApiPath, ) -> ApiResult> { let operations = state.operations.records.lock().await; let operation = operations @@ -625,28 +645,11 @@ async fn get_operation( Ok(Json(operation_response(&operation_id, operation))) } -enum ManualCompletion { - Split { - database: String, - container: String, - parent: u32, - mode: SplitMode, - split_epk: Epk, - operation: ManualControlPlaneOperation, - }, - Merge { - database: String, - container: String, - partitions: [u32; 2], - operation: ManualControlPlaneOperation, - }, -} - async fn advance_operation( State(state): State, - Path(operation_id): Path, + ApiPath(operation_id): ApiPath, ) -> ApiResult> { - let (completion, response) = { + let (kind, manual_operation, response) = { let mut operations = state.operations.records.lock().await; let operation = operations.get_mut(&operation_id).ok_or_else(|| { ApiError::not_found(format!("operation '{operation_id}' does not exist")) @@ -664,82 +667,13 @@ async fn advance_operation( )) })?; match action { - OperationAction::PendingSplit { - database, - container, - parent, - mode, - split_epk, - } => { - let manual_operation = state.emulator.store().begin_manual_split_partition( - &database, - &container, - parent, - split_epk.clone(), - ); + OperationAction::Pending(kind) => { + let manual_operation = kind.begin_manual(&state.emulator.store()); operation.phase = OperationPhase::Swapping; - operation.action = Some(OperationAction::ActiveSplit { - database, - container, - parent, - mode, - split_epk, - operation: manual_operation, - }); + operation.action = Some(OperationAction::Active(kind, manual_operation)); return Ok(Json(operation_response(&operation_id, operation))); } - OperationAction::PendingMerge { - database, - container, - partitions, - } => { - let manual_operation = state.emulator.store().begin_manual_merge_partitions( - &database, - &container, - partitions[0], - partitions[1], - ); - operation.phase = OperationPhase::Swapping; - operation.action = Some(OperationAction::ActiveMerge { - database, - container, - partitions, - operation: manual_operation, - }); - return Ok(Json(operation_response(&operation_id, operation))); - } - OperationAction::ActiveSplit { - database, - container, - parent, - mode, - split_epk, - operation: manual_operation, - } => ( - ManualCompletion::Split { - database, - container, - parent, - mode, - split_epk, - operation: manual_operation, - }, - response, - ), - OperationAction::ActiveMerge { - database, - container, - partitions, - operation: manual_operation, - } => ( - ManualCompletion::Merge { - database, - container, - partitions, - operation: manual_operation, - }, - response, - ), + OperationAction::Active(kind, manual_operation) => (kind, manual_operation, response), } }; let operations = state.operations.clone(); @@ -747,27 +681,7 @@ async fn advance_operation( let completion_id = operation_id.clone(); tokio::spawn(async move { let mut records = operations.records.lock().await; - let outcome = match completion { - ManualCompletion::Split { - database, - container, - parent, - mode, - split_epk, - operation, - } => { - complete_split( - &store, database, container, parent, mode, split_epk, operation, - ) - .await - } - ManualCompletion::Merge { - database, - container, - partitions, - operation, - } => complete_merge(&store, database, container, partitions, operation).await, - }; + let outcome = kind.complete(&store, manual_operation).await; finish_operation_locked(&mut records, &completion_id, outcome); }); Ok(Json(response)) @@ -780,7 +694,7 @@ struct EnabledRequest { async fn set_per_partition_failover( State(state): State, - Json(request): Json, + ApiJson(request): ApiJson, ) -> Json { state .emulator @@ -792,7 +706,7 @@ async fn set_per_partition_failover( async fn pause_replication( State(state): State, - Path(region): Path, + ApiPath(region): ApiPath, ) -> ApiResult> { ensure_region(&state.emulator.store(), ®ion)?; state.emulator.store().pause_replication(®ion); @@ -801,7 +715,7 @@ async fn pause_replication( async fn resume_replication( State(state): State, - Path(region): Path, + ApiPath(region): ApiPath, ) -> ApiResult> { ensure_region(&state.emulator.store(), ®ion)?; state.emulator.store().resume_replication(®ion); @@ -908,12 +822,53 @@ impl IntoResponse for ApiError { } } +/// A [`Path`] extractor whose rejection is shaped like every other error this +/// API returns (`{"error": "..."}"`), instead of axum's default plaintext +/// rejection body. +struct ApiPath(T); + +impl FromRequestParts for ApiPath +where + T: DeserializeOwned + Send + 'static, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + Path::::from_request_parts(parts, state) + .await + .map(|Path(value)| Self(value)) + .map_err(|rejection| ApiError::bad_request(rejection.to_string())) + } +} + +/// A [`Json`] extractor whose rejection is shaped like every other error this +/// API returns, instead of axum's default plaintext rejection body. +struct ApiJson(T); + +impl FromRequest for ApiJson +where + T: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = ApiError; + + async fn from_request(request: Request, state: &S) -> Result { + Json::::from_request(request, state) + .await + .map(|Json(value)| Self(value)) + .map_err(|rejection| ApiError::bad_request(rejection.to_string())) + } +} + #[cfg(test)] mod tests { use super::*; use azure_core::http::{headers::HeaderValue, Method, Request}; use azure_data_cosmos_driver::{ - in_memory_emulator::{ContainerConfig, VirtualAccountConfig, VirtualRegion}, + in_memory_emulator::{ + ContainerConfig, ReplicationConfig, VirtualAccountConfig, VirtualRegion, + }, models::PartitionKeyDefinition, }; use url::Url; @@ -962,7 +917,7 @@ mod tests { expected: &str, ) -> serde_json::Value { for _ in 0..100 { - let operation = get_operation(State(state.clone()), Path(operation_id.to_owned())) + let operation = get_operation(State(state.clone()), ApiPath(operation_id.to_owned())) .await .unwrap(); if operation["phase"] == expected { @@ -1040,7 +995,7 @@ mod tests { assert_eq!(response.1["phase"], "Preparing"); let operation_id = response.1["operationId"].as_str().unwrap().to_owned(); - let swapping = advance_operation(State(state.clone()), Path(operation_id.clone())) + let swapping = advance_operation(State(state.clone()), ApiPath(operation_id.clone())) .await .unwrap(); assert_eq!(swapping["phase"], "Swapping"); @@ -1051,7 +1006,7 @@ mod tests { .is_empty()); let completion_started = - advance_operation(State(state.clone()), Path(operation_id.clone())) + advance_operation(State(state.clone()), ApiPath(operation_id.clone())) .await .unwrap(); assert_eq!(completion_started["phase"], "Swapping"); @@ -1062,7 +1017,7 @@ mod tests { assert_eq!(succeeded["children"].as_array().unwrap().len(), 2); assert!(!succeeded["splitEpk"].as_str().unwrap().is_empty()); - let queried = get_operation(State(state.clone()), Path(operation_id)) + let queried = get_operation(State(state.clone()), ApiPath(operation_id)) .await .unwrap(); assert_eq!(queried["phase"], "Succeeded"); @@ -1092,8 +1047,8 @@ mod tests { .collect::>(); let merge = merge_partitions( State(state.clone()), - Path(("testdb".to_owned(), "testcoll".to_owned())), - Json(MergeRequest { + ApiPath(("testdb".to_owned(), "testcoll".to_owned())), + ApiJson(MergeRequest { partition_ids: [children[0], children[1]], progression_mode: ProgressionMode::Manual, lock_duration_ms: None, @@ -1102,12 +1057,12 @@ mod tests { .await .unwrap(); let merge_id = merge.1["operationId"].as_str().unwrap().to_owned(); - let merge_swapping = advance_operation(State(state.clone()), Path(merge_id.clone())) + let merge_swapping = advance_operation(State(state.clone()), ApiPath(merge_id.clone())) .await .unwrap(); assert_eq!(merge_swapping["phase"], "Swapping"); let merge_completion_started = - advance_operation(State(state.clone()), Path(merge_id.clone())) + advance_operation(State(state.clone()), ApiPath(merge_id.clone())) .await .unwrap(); assert_eq!(merge_completion_started["phase"], "Swapping"); @@ -1274,4 +1229,222 @@ mod tests { server.abort(); } + + #[tokio::test] + async fn storage_split_mode_round_trips_over_http() { + let gateway_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let account = + VirtualAccountConfig::new(vec![VirtualRegion::new("East US", gateway_url.clone())]) + .unwrap(); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(account)); + emulator.store().create_database("testdb"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + emulator.store().create_container_with_config( + "testdb", + "testcoll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + // `storage` mode requires documents in at least two distinct EPK + // groups to compute a balancing boundary. + for (id, pk) in [("1", "a"), ("2", "z")] { + let mut request = Request::new( + gateway_url.join("dbs/testdb/colls/testcoll/docs").unwrap(), + Method::Post, + ); + request.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from(format!(r#"["{pk}"]"#)), + ); + request + .set_body(serde_json::to_vec(&serde_json::json!({ "id": id, "pk": pk })).unwrap()); + let response = emulator.execute_request(&request).await.unwrap(); + assert_eq!(response.status(), azure_core::http::StatusCode::Created); + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = router( + emulator, + "test-account".to_owned(), + Vec::new(), + Arc::new(HostMetrics::default()), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let (status, created) = http_json( + address, + "POST", + "/databases/testdb/containers/testcoll/partitions/0/split", + Some(r#"{"mode":"storage"}"#), + ) + .await; + assert_eq!(status, 202); + let operation_id = created["operationId"].as_str().unwrap(); + let operation_path = format!("/operations/{operation_id}"); + + let mut terminal = None; + for _ in 0..100 { + let (status, operation) = http_json(address, "GET", &operation_path, None).await; + assert_eq!(status, 200); + if operation["phase"] == "Succeeded" { + terminal = Some(operation); + break; + } + tokio::task::yield_now().await; + } + let terminal = terminal.expect("storage split did not reach Succeeded"); + assert_eq!(terminal["mode"], "storage"); + assert_eq!(terminal["children"].as_array().unwrap().len(), 2); + assert!(!terminal["splitEpk"].as_str().unwrap().is_empty()); + + server.abort(); + } + + #[tokio::test] + async fn per_partition_failover_toggle_round_trips_over_http() { + let gateway_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let account = + VirtualAccountConfig::new(vec![VirtualRegion::new("East US", gateway_url)]).unwrap(); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(account)); + assert!(!emulator.store().config().per_partition_failover_enabled()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = router( + emulator.clone(), + "test-account".to_owned(), + Vec::new(), + Arc::new(HostMetrics::default()), + ); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + + let (status, enabled) = http_json( + address, + "PUT", + "/config/per-partition-failover", + Some(r#"{"enabled":true}"#), + ) + .await; + assert_eq!(status, 200); + assert_eq!(enabled["enabled"], true); + assert!(emulator.store().config().per_partition_failover_enabled()); + + let (status, disabled) = http_json( + address, + "PUT", + "/config/per-partition-failover", + Some(r#"{"enabled":false}"#), + ) + .await; + assert_eq!(status, 200); + assert_eq!(disabled["enabled"], false); + assert!(!emulator.store().config().per_partition_failover_enabled()); + + server.abort(); + } + + #[tokio::test] + async fn replication_pause_and_resume_round_trip_through_management_api() { + let east_url = Url::parse("http://127.0.0.1:18081/").unwrap(); + let west_url = Url::parse("http://127.0.0.1:18082/").unwrap(); + let account = VirtualAccountConfig::new(vec![ + VirtualRegion::new("East US", east_url.clone()), + VirtualRegion::new("West US", west_url.clone()), + ]) + .unwrap() + .with_replication_config(ReplicationConfig::fixed(Duration::from_millis(20))); + let emulator = Arc::new(InMemoryEmulatorHttpClient::new(account)); + emulator.store().create_database("testdb"); + let partition_key: PartitionKeyDefinition = serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], "kind": "Hash", "version": 2 + })) + .unwrap(); + emulator.store().create_container_with_config( + "testdb", + "testcoll", + partition_key, + ContainerConfig::new() + .with_partition_count(1) + .build() + .unwrap(), + ); + + let state = ManagementState { + emulator: emulator.clone(), + account_id: "test-account".into(), + bindings: Vec::::new().into(), + metrics: Arc::new(HostMetrics::default()), + operations: Arc::new(OperationRegistry::default()), + }; + + let paused = pause_replication(State(state.clone()), ApiPath("West US".to_owned())) + .await + .unwrap(); + assert_eq!(paused.0["replication"], "paused"); + + let mut create = Request::new( + east_url.join("dbs/testdb/colls/testcoll/docs").unwrap(), + Method::Post, + ); + create.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + create.set_body( + serde_json::to_vec(&serde_json::json!({ "id": "item1", "pk": "pk1" })).unwrap(), + ); + let response = emulator.execute_request(&create).await.unwrap(); + assert_eq!(response.status(), azure_core::http::StatusCode::Created); + + // Give the (very short) configured replication delay plenty of time + // to elapse. Because replication to West US is paused, the write + // must still be sitting in that region's buffer, not applied. + tokio::time::sleep(Duration::from_millis(200)).await; + let mut read_while_paused = Request::new( + west_url + .join("dbs/testdb/colls/testcoll/docs/item1") + .unwrap(), + Method::Get, + ); + read_while_paused.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + let response = emulator.execute_request(&read_while_paused).await.unwrap(); + assert_eq!( + response.status(), + azure_core::http::StatusCode::NotFound, + "paused replication must buffer the write instead of applying it" + ); + + let resumed = resume_replication(State(state.clone()), ApiPath("West US".to_owned())) + .await + .unwrap(); + assert_eq!(resumed.0["replication"], "resumed"); + emulator.store().drain_pending_replications().await; + + let mut read_after_resume = Request::new( + west_url + .join("dbs/testdb/colls/testcoll/docs/item1") + .unwrap(), + Method::Get, + ); + read_after_resume.headers_mut().insert( + "x-ms-documentdb-partitionkey", + HeaderValue::from_static(r#"["pk1"]"#), + ); + let response = emulator.execute_request(&read_after_resume).await.unwrap(); + assert_eq!( + response.status(), + azure_core::http::StatusCode::Ok, + "resuming replication must apply the buffered write" + ); + } } diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 index 5f1e93fb073..267a2ebbf0a 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestCleanup.ps1 @@ -10,7 +10,8 @@ $ShutdownTimeout = 30 if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { if ($env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) { $hostProcess = Get-Process -Id ([int]$env:AZURE_COSMOS_INMEMORY_EMULATOR_PID) -ErrorAction SilentlyContinue - } else { + } + else { $hostProcess = Get-Process azure_data_cosmos_emulator -ErrorAction SilentlyContinue } if ($hostProcess) { @@ -23,7 +24,8 @@ elseif ($IsWindows) { $EmulatorPath = & "$PSScriptRoot\Get-CosmosEmulatorPath.ps1" if ($null -eq $EmulatorPath) { Write-Host "Unable to confirm Cosmos DB Emulator location, skipping shutdown." - } else { + } + else { Write-Host "Shutting down Cosmos DB Emulator at '$EmulatorPath'." & $EmulatorPath /shutdown @@ -46,12 +48,14 @@ elseif ($IsWindows) { Write-Warning "Cosmos DB Emulator did not shut down within ${ShutdownTimeout} seconds." } } -} elseif (Get-Command docker -ErrorAction SilentlyContinue) { +} +elseif (Get-Command docker -ErrorAction SilentlyContinue) { $containerName = "cosmosdb-emulator-test" $containerStatus = docker ps -a --filter "name=$containerName" --format "{{.Status}}" if (-not $containerStatus) { Write-Host "No Cosmos DB Emulator container found, skipping cleanup." - } else { + } + else { Write-Host "Stopping and removing Cosmos DB Emulator container '$containerName'." Invoke-LoggedCommand "docker rm -f $containerName" @@ -74,7 +78,8 @@ elseif ($IsWindows) { Write-Warning "Cosmos DB Emulator did not shut down within ${ShutdownTimeout} seconds." } } -} else { +} +else { Write-Host "No Cosmos DB Emulator found to clean up." } diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 4c21b0f8671..5a467577dea 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -21,7 +21,8 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $repoRoot = (Resolve-Path ([System.IO.Path]::Combine($PSScriptRoot, '..', '..', '..', '..'))).Path $configuration = if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'inmemory-v2') { [System.IO.Path]::Combine($repoRoot, 'sdk', 'cosmos', 'azure_data_cosmos_emulator', 'config', 'ci-gateway-v2.json') - } else { + } + else { [System.IO.Path]::Combine($repoRoot, 'sdk', 'cosmos', 'azure_data_cosmos_emulator', 'config', 'ci-gateway-v1.json') } $ready = $false @@ -34,7 +35,8 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $response = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -ErrorAction Stop $health = $response.Content | ConvertFrom-Json $ready = $response.StatusCode -eq 200 -and $health.gateway20Enabled -eq $expectedGateway20 - } catch { + } + catch { $ready = $false } } @@ -49,14 +51,16 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { try { Invoke-LoggedCommand 'cargo test -p azure_data_cosmos_emulator --all-features' Invoke-LoggedCommand 'cargo build -p azure_data_cosmos_emulator' - } finally { + } + finally { Pop-Location } LogGroupEnd $executableName = if ($IsWindows) { 'azure_data_cosmos_emulator.exe' - } else { + } + else { 'azure_data_cosmos_emulator' } $executable = [System.IO.Path]::Combine($repoRoot, 'target', 'debug', $executableName) @@ -89,13 +93,19 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $readyRecord = $candidate $managementEndpoint = [string]$readyRecord.managementEndpoint $accountEndpoint = [string]$readyRecord.accountEndpoint - $hasGateway20 = @($readyRecord.regions | Where-Object { $_.gateway20Endpoint }).Count -gt 0 + # V1 omits this optional property; direct access throws under strict mode. + $hasGateway20 = @($readyRecord.regions | Where-Object { + $gateway20Endpoint = $_.PSObject.Properties['gateway20Endpoint'] + $null -ne $gateway20Endpoint -and + -not [string]::IsNullOrWhiteSpace([string]$gateway20Endpoint.Value) + }).Count -gt 0 if (-not $managementEndpoint -or -not $accountEndpoint -or $hasGateway20 -ne $expectedGateway20) { throw 'Hosted emulator ready record does not match the requested gateway mode.' } $healthUrl = ([System.Uri]::new([System.Uri]$managementEndpoint, 'health')).AbsoluteUri } - } catch { + } + catch { $readyRecord = $null Write-Host "Waiting for a valid hosted emulator ready record: $($_.Exception.Message)" } @@ -109,7 +119,8 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $ready = $true break } - } catch { + } + catch { Write-Host "Waiting for hosted in-memory emulator readiness: $($_.Exception.Message)" } } @@ -164,7 +175,8 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'vnext') { $existing = docker ps --filter "name=$vnextContainerName" --format "{{.Names}}" 2>$null if ($existing -eq $vnextContainerName) { Write-Host "Cosmos DB vnext emulator container '$vnextContainerName' is already running." - } else { + } + else { LogGroupStart "Starting Cosmos DB vnext emulator (Docker)" Invoke-LoggedCommand "docker pull $vnextImage" Invoke-LoggedCommand "docker run -d --name $vnextContainerName --publish 8081:8081 --publish 8080:8080 -e ENABLE_EXPLORER=false $vnextImage" @@ -182,7 +194,8 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -eq 'vnext') { $ready = $true break } - } catch { + } + catch { Write-Host "Waiting for vnext emulator readiness: $($_.Exception.Message)" } Start-Sleep -Seconds 5 @@ -215,7 +228,8 @@ if ($IsAzDo) { # We only run Cosmos DB tests on Windows agents in Azure DevOps if ($IsWindows) { $env:AZURE_COSMOS_TEST_MODE = "required" - } else { + } + else { $env:AZURE_COSMOS_TEST_MODE = "skipped" Write-Host "Skipping Cosmos DB Emulator setup on non-Windows Azure DevOps agents." return @@ -228,7 +242,8 @@ if ($IsWindows) { $EmulatorPath = & "$PSScriptRoot\Get-CosmosEmulatorPath.ps1" if ($null -ne $EmulatorPath) { Write-Host "Found Cosmos DB Emulator at '$EmulatorPath'. Skipping Cosmos DB Emulator install." - } else { + } + else { LogGroupStart "Installing Cosmos DB Emulator" & "$PSScriptRoot\..\..\..\..\eng\common\scripts\Cosmos-Emulator.ps1" ` -StartParameters "/noexplorer /noui /enablepreview /EnableSqlComputeEndpoint /SqlComputePort=9999 /disableratelimiting /partitioncount=50 /consistency=Strong /enableaadauthentication" ` @@ -254,25 +269,26 @@ if ($IsWindows) { $response = Invoke-WebRequest -Uri $emulatorUrl -SkipCertificateCheck -UseBasicParsing -ErrorAction Stop Write-Host "Emulator responded with status $($response.StatusCode)." $emulatorReady = $true - } catch { - # Some exceptions (e.g. connection refused) have no Response property. - $response = $null - if ($_.Exception.PSObject.Properties['Response']) { - $response = $_.Exception.Response - } - if ($null -ne $response -and $null -ne $response.StatusCode) { - $statusCode = $response.StatusCode.value__ - if ($statusCode -ge 400 -and $statusCode -lt 500) { - # 4xx means the emulator is up but rejecting unauthenticated requests - Write-Host "Emulator responded with status $statusCode (expected auth failure). Emulator is ready." - $emulatorReady = $true - continue - } - } - # No HTTP response or non-4xx status: treat as retryable failure - $probeRetry++ - Write-Host "[Retry: $probeRetry/$maxProbeRetries] Emulator not yet responding. Exception: $($_.Exception.Message)" - Start-Sleep -Seconds 5 + } + catch { + # Some exceptions (e.g. connection refused) have no Response property. + $response = $null + if ($_.Exception.PSObject.Properties['Response']) { + $response = $_.Exception.Response + } + if ($null -ne $response -and $null -ne $response.StatusCode) { + $statusCode = $response.StatusCode.value__ + if ($statusCode -ge 400 -and $statusCode -lt 500) { + # 4xx means the emulator is up but rejecting unauthenticated requests + Write-Host "Emulator responded with status $statusCode (expected auth failure). Emulator is ready." + $emulatorReady = $true + continue + } + } + # No HTTP response or non-4xx status: treat as retryable failure + $probeRetry++ + Write-Host "[Retry: $probeRetry/$maxProbeRetries] Emulator not yet responding. Exception: $($_.Exception.Message)" + Start-Sleep -Seconds 5 } } if (-not $emulatorReady) { @@ -288,14 +304,16 @@ if ($IsWindows) { # Run tests single-threaded to avoid env var contamination from proxy tests. $env:RUST_TEST_THREADS = "1" -} elseif (Get-Command "docker" -ErrorAction SilentlyContinue) { +} +elseif (Get-Command "docker" -ErrorAction SilentlyContinue) { Write-Host "Docker detected. Using Cosmos DB Emulator in Docker." # Check if the emulator is already running $existingContainer = docker ps --filter "name=cosmosdb-emulator-test" --format "{{.Names}}" if ($existingContainer -eq "cosmosdb-emulator-test") { Write-Host "Cosmos DB Emulator container is already running." - } else { + } + else { LogGroupStart "Starting Cosmos DB Emulator in Docker" # Start Cosmos DB Emulator in Docker $containerName = "cosmosdb-emulator-test" @@ -313,10 +331,12 @@ if ($IsWindows) { $emulatorStarted = $true Write-Host "Cosmos DB Emulator started successfully." break - } elseif ($lastLine -match "^\s*Started (\d+/\d+) partitions\s*$") { + } + elseif ($lastLine -match "^\s*Started (\d+/\d+) partitions\s*$") { $partitionsStarted = $matches[1] Write-Host "[Retry: $retryCount/$maxRetries] Emulator still starting, $partitionsStarted partitions started." - } else { + } + else { Write-Host "[Retry: $retryCount/$maxRetries] Emulator still starting" } $retryCount++ @@ -339,7 +359,8 @@ if ($IsWindows) { $env:RUST_TEST_THREADS = "1" Write-Host "Cosmos DB Emulator is running in Docker." -} else { +} +else { # We're running a local build or we're on a macOS agent. # We can't run the emulator on the macOS agent, and we don't want to fail local builds because the emulator isn't installed. Write-Host "Cosmos DB Emulator is not available on this platform. Skipping test setup." diff --git a/sdk/cosmos/inmemory-emulator-matrix.json b/sdk/cosmos/inmemory-emulator-matrix.json index 0eb56c7779d..99197acfc76 100644 --- a/sdk/cosmos/inmemory-emulator-matrix.json +++ b/sdk/cosmos/inmemory-emulator-matrix.json @@ -10,7 +10,12 @@ "Pool": "env:LINUXPOOL" } }, - "RustToolchainName": ["stable"], - "AZURE_COSMOS_EMULATOR_FLAVOR": ["inmemory-v1", "inmemory-v2"] + "RustToolchainName": [ + "stable" + ], + "AZURE_COSMOS_EMULATOR_FLAVOR": [ + "inmemory-v1", + "inmemory-v2" + ] } -} +} \ No newline at end of file diff --git a/sdk/cosmos/vnext-emulator-matrix.json b/sdk/cosmos/vnext-emulator-matrix.json index 13e5bd59857..ce9bcb029e7 100644 --- a/sdk/cosmos/vnext-emulator-matrix.json +++ b/sdk/cosmos/vnext-emulator-matrix.json @@ -9,7 +9,11 @@ "Pool": "env:LINUXPOOL" } }, - "RustToolchainName": ["stable"], - "AZURE_COSMOS_EMULATOR_FLAVOR": ["vnext"] + "RustToolchainName": [ + "stable" + ], + "AZURE_COSMOS_EMULATOR_FLAVOR": [ + "vnext" + ] } -} +} \ No newline at end of file From d4571f8ba859dae05a0a55e24b230c0c3a0872bf Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 13:46:35 +0000 Subject: [PATCH 33/44] Fix emulator CI readiness Handle the optional Gateway 2.0 endpoint safely when parsing Gateway V1 ready records under strict PowerShell mode. Replace unsupported local README links with plain repository paths. --- sdk/cosmos/azure_data_cosmos_emulator/README.md | 14 +++++++------- sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_emulator/README.md b/sdk/cosmos/azure_data_cosmos_emulator/README.md index e45d31f27fe..3e2e4a5ddf6 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/README.md +++ b/sdk/cosmos/azure_data_cosmos_emulator/README.md @@ -9,8 +9,8 @@ it over the Cosmos DB wire protocol. This is an **SDK engineering and test tool**, not a supported customer product: it provides no service compatibility, durability, performance, or support guarantees. See -[`docs/adr/001_build_memory_backed_sdk_test_emulator.md`](docs/adr/001_build_memory_backed_sdk_test_emulator.md) -for the full rationale and scope, and [`docs/plan.md`](docs/plan.md) for the +`docs/adr/001_build_memory_backed_sdk_test_emulator.md` +for the full rationale and scope, and `docs/plan.md` for the complete design (configuration schema, management REST API, Gateway 2.0 support, and CI integration). @@ -28,7 +28,7 @@ support, and CI integration). ## Quick start Build and run against one of the sample configs checked into -[`config/`](config/): +`config/`: ```sh cargo build -p azure_data_cosmos_emulator @@ -61,9 +61,9 @@ curl -X POST "http://127.0.0.1:49150/databases/testdb/containers/testcoll/partit A single JSON file (`--config`) describes the account topology, the databases/containers to create, and optional seed items — all applied on -startup. See [`docs/plan.md`](docs/plan.md#4-configuration-file) for the full -field reference; [`config/ci-gateway-v1.json`](config/ci-gateway-v1.json) and -[`config/ci-gateway-v2.json`](config/ci-gateway-v2.json) are minimal, +startup. See `docs/plan.md#4-configuration-file` for the full field +reference; `config/ci-gateway-v1.json` and +`config/ci-gateway-v2.json` are minimal, CI-oriented examples (both use port `0` for every listener, so the OS assigns free ports and the resolved endpoints come from the `ready` record). @@ -77,6 +77,6 @@ The hosted emulator is also exercised end-to-end by `azure_data_cosmos_driver`'s and `azure_data_cosmos`'s existing emulator test suites, gated behind `test_category = "emulator_inmemory"` (and `"emulator_inmemory_gateway_v2"` for Gateway 2.0-specific tests). See -[`sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1`](../eng/scripts/Invoke-CosmosTestSetup.ps1) +`sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1` for how CI builds, starts, and health-checks the host process before running those suites. diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 5a467577dea..296276565fb 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -95,10 +95,10 @@ if ($env:AZURE_COSMOS_EMULATOR_FLAVOR -in @('inmemory-v1', 'inmemory-v2')) { $accountEndpoint = [string]$readyRecord.accountEndpoint # V1 omits this optional property; direct access throws under strict mode. $hasGateway20 = @($readyRecord.regions | Where-Object { - $gateway20Endpoint = $_.PSObject.Properties['gateway20Endpoint'] - $null -ne $gateway20Endpoint -and + $gateway20Endpoint = $_.PSObject.Properties['gateway20Endpoint'] + $null -ne $gateway20Endpoint -and -not [string]::IsNullOrWhiteSpace([string]$gateway20Endpoint.Value) - }).Count -gt 0 + }).Count -gt 0 if (-not $managementEndpoint -or -not $accountEndpoint -or $hasGateway20 -ne $expectedGateway20) { throw 'Hosted emulator ready record does not match the requested gateway mode.' } From 1868146ea691d71c6db7199a509bb05175ca8b33 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 14:27:31 +0000 Subject: [PATCH 34/44] Fix link verification issues --- .../driver/transport/gateway_v2_dispatch.rs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index a0055d5cb58..6d4c783a03a 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -403,6 +403,21 @@ pub(crate) fn wrap_request_for_gateway_v2( }) } +// Header names consulted as an outer-HTTP fallback in +// `unwrap_response_for_gateway_v2` below when the RNTBD response frame +// itself doesn't carry the corresponding field. `HeaderName::from_static` +// does no per-call work (it just wraps the `&'static str`), but there's no +// reason to re-derive these six on every response either, so build the +// list once as a `static` rather than as a fresh array literal per call. +static OUTER_FALLBACK_HEADERS: [HeaderName; 6] = [ + HeaderName::from_static(response_header_names::SERVER_DURATION_MS), + HeaderName::from_static(response_header_names::LSN), + HeaderName::from_static(response_header_names::ITEM_LSN), + X_MS_GLOBAL_COMMITTED_LSN, + HeaderName::from_static(response_header_names::QUERY_METRICS), + HeaderName::from_static(response_header_names::INDEX_METRICS), +]; + /// Decodes a Gateway 2.0 RNTBD response body into a synthetic HTTP response. pub(crate) fn unwrap_response_for_gateway_v2( response: HttpResponse, @@ -410,23 +425,15 @@ pub(crate) fn unwrap_response_for_gateway_v2( // Only these headers are ever consulted as an outer-HTTP fallback below, // so capture just their (at most six) values instead of cloning the // entire response header map up front. - let outer_fallbacks: Vec<(HeaderName, String)> = [ - response_header_names::SERVER_DURATION_MS, - response_header_names::LSN, - response_header_names::ITEM_LSN, - response_header_names::GLOBAL_COMMITTED_LSN, - response_header_names::QUERY_METRICS, - response_header_names::INDEX_METRICS, - ] - .into_iter() - .filter_map(|name| { - let header = HeaderName::from_static(name); - response - .headers - .get_optional_str(&header) - .map(|value| (header, value.to_owned())) - }) - .collect(); + let outer_fallbacks: Vec<(HeaderName, String)> = OUTER_FALLBACK_HEADERS + .iter() + .filter_map(|header| { + response + .headers + .get_optional_str(header) + .map(|value| (header.clone(), value.to_owned())) + }) + .collect(); let response = RntbdResponse::read(&response.body)?; let status = u16::from(response.status.status_code()); if !(100..=599).contains(&status) { From c1dc1273457487b91476439431907b8f9bd3097c Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 20:44:15 +0000 Subject: [PATCH 35/44] Update gateway_v2_dispatch.rs --- .../driver/transport/gateway_v2_dispatch.rs | 122 +++++++++++++++++- 1 file changed, 118 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index 6d4c783a03a..edd0e8c9869 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -28,7 +28,7 @@ use crate::{ use super::{ cosmos_headers::SUPPORTED_CAPABILITIES_BITS, cosmos_transport_client::{HttpRequest, HttpResponse}, - rntbd::{RntbdRequestFrame, RntbdResponse, Token}, + rntbd::{tokens::RntbdRequestToken, RntbdRequestFrame, RntbdResponse, Token}, AuthorizationContext, }; @@ -318,6 +318,25 @@ pub(crate) fn wrap_request_for_gateway_v2( metadata.push(Token::match_condition(value.to_owned())); } + // Debug-only visibility into exactly which RNTBD metadata tokens are + // about to go on the wire. Logs token IDs (resolved to their symbolic + // name where recognized), never values — `AuthorizationToken` carries + // the request's HMAC signature, which must never be logged. Enable with + // e.g. `RUST_LOG=azure_data_cosmos_driver=debug`; the field expressions + // below are only evaluated when the level/target is actually enabled. + tracing::debug!( + operation = ?inputs.operation_type, + resource_type = ?inputs.resource_type, + tokens = ?metadata + .iter() + .map(|token| match RntbdRequestToken::try_from(token.id.value()) { + Ok(name) => format!("{name:?}"), + Err(()) => format!("{:#06x}", token.id.value()), + }) + .collect::>(), + "Gateway 2.0 request RNTBD metadata tokens" + ); + let frame = RntbdRequestFrame { resource_type: inputs.resource_type, operation_type: inputs.operation_type, @@ -407,15 +426,22 @@ pub(crate) fn wrap_request_for_gateway_v2( // `unwrap_response_for_gateway_v2` below when the RNTBD response frame // itself doesn't carry the corresponding field. `HeaderName::from_static` // does no per-call work (it just wraps the `&'static str`), but there's no -// reason to re-derive these six on every response either, so build the +// reason to re-derive these seven on every response either, so build the // list once as a `static` rather than as a fresh array literal per call. -static OUTER_FALLBACK_HEADERS: [HeaderName; 6] = [ +// +// `ITEM_COUNT` belongs here alongside `QUERY_METRICS`/`INDEX_METRICS`: like +// them, it is page-level query-engine metadata the RNTBD frame schema has no +// token for (see `RntbdResponseToken`), so Gateway 2.0 surfaces it only as an +// outer HTTP header. Omitting it here silently dropped `x-ms-item-count` on +// every Gateway 2.0 query response. +static OUTER_FALLBACK_HEADERS: [HeaderName; 7] = [ HeaderName::from_static(response_header_names::SERVER_DURATION_MS), HeaderName::from_static(response_header_names::LSN), HeaderName::from_static(response_header_names::ITEM_LSN), X_MS_GLOBAL_COMMITTED_LSN, HeaderName::from_static(response_header_names::QUERY_METRICS), HeaderName::from_static(response_header_names::INDEX_METRICS), + HeaderName::from_static(response_header_names::ITEM_COUNT), ]; /// Decodes a Gateway 2.0 RNTBD response body into a synthetic HTTP response. @@ -423,7 +449,7 @@ pub(crate) fn unwrap_response_for_gateway_v2( response: HttpResponse, ) -> azure_core::Result { // Only these headers are ever consulted as an outer-HTTP fallback below, - // so capture just their (at most six) values instead of cloning the + // so capture just their (at most seven) values instead of cloning the // entire response header map up front. let outer_fallbacks: Vec<(HeaderName, String)> = OUTER_FALLBACK_HEADERS .iter() @@ -2046,6 +2072,94 @@ mod tests { ); } + #[test] + fn unwrap_applies_outer_http_fallback_headers_missing_from_rntbd_frame() { + // None of these seven are ever emitted as RNTBD response tokens (see + // `RntbdResponseToken`), so an empty token list forces every value in + // the assertions below to come from the outer HTTP headers captured + // before the frame is parsed, not from `RntbdResponse::read`. + let activity_id = Uuid::parse_str(ACTIVITY_ID).unwrap(); + let mut headers = Headers::new(); + headers.insert( + HeaderName::from_static(response_header_names::SERVER_DURATION_MS), + "12.5", + ); + headers.insert(HeaderName::from_static(response_header_names::LSN), "7"); + headers.insert( + HeaderName::from_static(response_header_names::ITEM_LSN), + "8", + ); + headers.insert(X_MS_GLOBAL_COMMITTED_LSN, "9"); + headers.insert( + HeaderName::from_static(response_header_names::QUERY_METRICS), + "metrics-blob", + ); + headers.insert( + HeaderName::from_static(response_header_names::INDEX_METRICS), + "index-blob", + ); + headers.insert( + HeaderName::from_static(response_header_names::ITEM_COUNT), + "1", + ); + + let response = HttpResponse { + status: 200, + headers, + body: response_frame(200, activity_id, |_| {}, b"{}"), + }; + + let unwrapped = unwrap_response_for_gateway_v2(response).unwrap(); + + assert_eq!( + unwrapped.headers.get_optional_str(&HeaderName::from_static( + response_header_names::SERVER_DURATION_MS + )), + Some("12.5") + ); + assert_eq!( + unwrapped + .headers + .get_optional_str(&HeaderName::from_static(response_header_names::LSN)), + Some("7") + ); + assert_eq!( + unwrapped + .headers + .get_optional_str(&HeaderName::from_static(response_header_names::ITEM_LSN)), + Some("8") + ); + assert_eq!( + unwrapped + .headers + .get_optional_str(&X_MS_GLOBAL_COMMITTED_LSN), + Some("9") + ); + assert_eq!( + unwrapped.headers.get_optional_str(&HeaderName::from_static( + response_header_names::QUERY_METRICS + )), + Some("metrics-blob") + ); + assert_eq!( + unwrapped.headers.get_optional_str(&HeaderName::from_static( + response_header_names::INDEX_METRICS + )), + Some("index-blob") + ); + // Regression check for the outer-fallback list silently omitting + // `item_count`: Gateway 2.0 query responses carry `x-ms-item-count` + // only as an outer HTTP header (there is no RNTBD token for it), so a + // missing entry here previously made the driver report `item_count` + // as absent even when the service returned it. + assert_eq!( + unwrapped + .headers + .get_optional_str(&HeaderName::from_static(response_header_names::ITEM_COUNT)), + Some("1") + ); + } + #[test] fn unwrap_prefixes_session_token_with_partition_key_range_id() { let response = HttpResponse { From eeab9cf5cb73520c64e582cee15294fd331f8e9a Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 21:51:27 +0000 Subject: [PATCH 36/44] Update validation.rs --- .../in_memory_emulator_tests/validation.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs index 5a48662b8aa..2629cb48ae7 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs @@ -233,6 +233,21 @@ impl HeaderValidationSpec { /// in-memory emulator cannot faithfully reproduce. Query comparison tests /// should log those values for diagnostics, while using this spec for basic /// header sanity and presence checks. + /// + /// `item_count` uses `Exists` rather than `Symmetric`: a real backend is + /// free to interleave empty probe pages (0 items, but a continuation) with + /// non-empty ones while scanning a partition/range, even when the query is + /// guaranteed to eventually yield a document -- e.g. R1 (0 docs, `CT1`), + /// R2 (2 docs, `CT2`), R3 (1 doc, no continuation). Such empty pages may + /// omit `x-ms-item-count` entirely. The in-memory emulator has no + /// equivalent scan-in-progress state and always emits exactly one item + /// per page, so page *count* can coincidentally match between backends + /// (e.g. 3 real pages vs. 3 emulator pages) while page *boundaries* don't + /// align at all. Since `compare_page_headers_if_aligned` zips pages by + /// index once counts match, a `Symmetric` rule ends up comparing + /// unrelated pages (e.g. empty real page 0 vs. non-empty emulator page 0) + /// and produces false positives (e.g. `hash_partition_scope` in + /// `query_comparison.rs`). pub fn for_query_operation() -> Self { Self::new() .with_rule("activity_id", HeaderMatch::Exists) @@ -240,7 +255,7 @@ impl HeaderValidationSpec { .with_rule("session_token", HeaderMatch::Exists) .with_rule("etag", HeaderMatch::Symmetric) .with_rule("continuation", HeaderMatch::Symmetric) - .with_rule("item_count", HeaderMatch::Symmetric) + .with_rule("item_count", HeaderMatch::Exists) .with_rule("index_metrics", HeaderMatch::Ignore) .with_rule("query_metrics", HeaderMatch::Ignore) .with_rule("server_duration_ms", HeaderMatch::Exists) From 319728726faf5ab44741478faea073e005350f39 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 22:27:15 +0000 Subject: [PATCH 37/44] Fix build issue --- Cargo.lock | 2 +- sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1092c3e0fdd..fc2d782f096 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -661,7 +661,7 @@ name = "azure_data_cosmos_emulator" version = "0.1.0" dependencies = [ "axum 0.8.9", - "azure_core 1.1.0", + "azure_core 1.2.0-beta.1", "azure_data_cosmos_driver", "clap", "serde", diff --git a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml index 03c33ce6f0c..c84543cdf86 100644 --- a/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos_emulator/Cargo.toml @@ -11,7 +11,12 @@ rust-version.workspace = true [dependencies] axum.workspace = true -azure_core.workspace = true +# Path+version — `azure_data_cosmos_driver` (below) depends on the local, +# unreleased `azure_core` (see its Cargo.toml), so this crate must resolve +# the same local copy. Using `workspace = true` here would instead pull the +# published crate, producing two incompatible `AsyncRawResponse` types +# across the `data_plane`/`gateway_v2` boundary with `azure_data_cosmos_driver`. +azure_core = { path = "../../core/azure_core", version = "1.2.0-beta.1", default-features = false } azure_data_cosmos_driver = { path = "../azure_data_cosmos_driver", default-features = false, features = [ "rustls", "__internal_in_memory_emulator", From b5c941308f76cdcf0a956242d2a96437815dbb33 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 22:55:31 +0000 Subject: [PATCH 38/44] Fixes rntbd response tokens --- .../driver/transport/gateway_v2_dispatch.rs | 132 ++++-------------- .../src/driver/transport/rntbd/response.rs | 56 ++++++++ .../src/driver/transport/rntbd/tokens.rs | 18 ++- .../src/in_memory_emulator/gateway_v2.rs | 16 +-- 4 files changed, 101 insertions(+), 121 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index edd0e8c9869..7630b669fb6 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -422,44 +422,10 @@ pub(crate) fn wrap_request_for_gateway_v2( }) } -// Header names consulted as an outer-HTTP fallback in -// `unwrap_response_for_gateway_v2` below when the RNTBD response frame -// itself doesn't carry the corresponding field. `HeaderName::from_static` -// does no per-call work (it just wraps the `&'static str`), but there's no -// reason to re-derive these seven on every response either, so build the -// list once as a `static` rather than as a fresh array literal per call. -// -// `ITEM_COUNT` belongs here alongside `QUERY_METRICS`/`INDEX_METRICS`: like -// them, it is page-level query-engine metadata the RNTBD frame schema has no -// token for (see `RntbdResponseToken`), so Gateway 2.0 surfaces it only as an -// outer HTTP header. Omitting it here silently dropped `x-ms-item-count` on -// every Gateway 2.0 query response. -static OUTER_FALLBACK_HEADERS: [HeaderName; 7] = [ - HeaderName::from_static(response_header_names::SERVER_DURATION_MS), - HeaderName::from_static(response_header_names::LSN), - HeaderName::from_static(response_header_names::ITEM_LSN), - X_MS_GLOBAL_COMMITTED_LSN, - HeaderName::from_static(response_header_names::QUERY_METRICS), - HeaderName::from_static(response_header_names::INDEX_METRICS), - HeaderName::from_static(response_header_names::ITEM_COUNT), -]; - /// Decodes a Gateway 2.0 RNTBD response body into a synthetic HTTP response. pub(crate) fn unwrap_response_for_gateway_v2( response: HttpResponse, ) -> azure_core::Result { - // Only these headers are ever consulted as an outer-HTTP fallback below, - // so capture just their (at most seven) values instead of cloning the - // entire response header map up front. - let outer_fallbacks: Vec<(HeaderName, String)> = OUTER_FALLBACK_HEADERS - .iter() - .filter_map(|header| { - response - .headers - .get_optional_str(header) - .map(|value| (header.clone(), value.to_owned())) - }) - .collect(); let response = RntbdResponse::read(&response.body)?; let status = u16::from(response.status.status_code()); if !(100..=599).contains(&status) { @@ -528,10 +494,20 @@ pub(crate) fn unwrap_response_for_gateway_v2( if let Some(owner_full_name) = response.owner_full_name { headers.insert(response_header_names::OWNER_FULL_NAME, owner_full_name); } - for (header, value) in outer_fallbacks { - if headers.get_optional_str(&header).is_none() { - headers.insert(header, value); - } + if let Some(item_count) = response.item_count { + headers.insert(response_header_names::ITEM_COUNT, item_count.to_string()); + } + if let Some(query_metrics) = response.query_metrics { + headers.insert(response_header_names::QUERY_METRICS, query_metrics); + } + if let Some(index_utilization) = response.index_utilization { + headers.insert(response_header_names::INDEX_METRICS, index_utilization); + } + if let Some(request_duration_ms) = response.request_duration_ms { + headers.insert( + response_header_names::SERVER_DURATION_MS, + request_duration_ms.to_string(), + ); } Ok(HttpResponse { @@ -2008,6 +1984,10 @@ mod tests { write_i64_token(tokens, 0x0032, 43); write_i64_token(tokens, 0x0029, 44); write_string_token(tokens, 0x0017, "dbs/db1/colls/coll1/docs/doc1"); + write_u32_token(tokens, 0x0014, 5); + write_string_token(tokens, 0x0028, "metrics-blob"); + write_string_token(tokens, 0x0044, "index-blob"); + write_double_token(tokens, 0x0051, 12.5); }, b"{}", ), @@ -2070,70 +2050,11 @@ mod tests { .get_optional_str(&X_MS_GLOBAL_COMMITTED_LSN), Some("44") ); - } - - #[test] - fn unwrap_applies_outer_http_fallback_headers_missing_from_rntbd_frame() { - // None of these seven are ever emitted as RNTBD response tokens (see - // `RntbdResponseToken`), so an empty token list forces every value in - // the assertions below to come from the outer HTTP headers captured - // before the frame is parsed, not from `RntbdResponse::read`. - let activity_id = Uuid::parse_str(ACTIVITY_ID).unwrap(); - let mut headers = Headers::new(); - headers.insert( - HeaderName::from_static(response_header_names::SERVER_DURATION_MS), - "12.5", - ); - headers.insert(HeaderName::from_static(response_header_names::LSN), "7"); - headers.insert( - HeaderName::from_static(response_header_names::ITEM_LSN), - "8", - ); - headers.insert(X_MS_GLOBAL_COMMITTED_LSN, "9"); - headers.insert( - HeaderName::from_static(response_header_names::QUERY_METRICS), - "metrics-blob", - ); - headers.insert( - HeaderName::from_static(response_header_names::INDEX_METRICS), - "index-blob", - ); - headers.insert( - HeaderName::from_static(response_header_names::ITEM_COUNT), - "1", - ); - - let response = HttpResponse { - status: 200, - headers, - body: response_frame(200, activity_id, |_| {}, b"{}"), - }; - - let unwrapped = unwrap_response_for_gateway_v2(response).unwrap(); - - assert_eq!( - unwrapped.headers.get_optional_str(&HeaderName::from_static( - response_header_names::SERVER_DURATION_MS - )), - Some("12.5") - ); - assert_eq!( - unwrapped - .headers - .get_optional_str(&HeaderName::from_static(response_header_names::LSN)), - Some("7") - ); assert_eq!( unwrapped .headers - .get_optional_str(&HeaderName::from_static(response_header_names::ITEM_LSN)), - Some("8") - ); - assert_eq!( - unwrapped - .headers - .get_optional_str(&X_MS_GLOBAL_COMMITTED_LSN), - Some("9") + .get_optional_str(&HeaderName::from_static(response_header_names::ITEM_COUNT)), + Some("5") ); assert_eq!( unwrapped.headers.get_optional_str(&HeaderName::from_static( @@ -2147,16 +2068,11 @@ mod tests { )), Some("index-blob") ); - // Regression check for the outer-fallback list silently omitting - // `item_count`: Gateway 2.0 query responses carry `x-ms-item-count` - // only as an outer HTTP header (there is no RNTBD token for it), so a - // missing entry here previously made the driver report `item_count` - // as absent even when the service returned it. assert_eq!( - unwrapped - .headers - .get_optional_str(&HeaderName::from_static(response_header_names::ITEM_COUNT)), - Some("1") + unwrapped.headers.get_optional_str(&HeaderName::from_static( + response_header_names::SERVER_DURATION_MS + )), + Some("12.5") ); } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs index d726f70379c..edc7b747969 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/response.rs @@ -48,6 +48,14 @@ pub(crate) struct RntbdResponse { pub(crate) transport_request_id: Option, /// Session token for session consistency. pub(crate) session_token: Option, + /// Item count for feed-style (query/read-feed) responses. + pub(crate) item_count: Option, + /// Serialized query execution metrics. + pub(crate) query_metrics: Option, + /// Serialized index utilization/advice metrics. + pub(crate) index_utilization: Option, + /// Backend request duration in milliseconds. + pub(crate) request_duration_ms: Option, } impl RntbdResponse { @@ -131,6 +139,30 @@ impl RntbdResponse { TokenValue::String(value.clone()), )); } + if let Some(value) = self.item_count { + metadata.push(Token::new( + RntbdResponseToken::ItemCount, + TokenValue::ULong(value), + )); + } + if let Some(value) = &self.query_metrics { + metadata.push(Token::new( + RntbdResponseToken::QueryMetrics, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = &self.index_utilization { + metadata.push(Token::new( + RntbdResponseToken::IndexUtilization, + TokenValue::String(value.clone()), + )); + } + if let Some(value) = self.request_duration_ms { + metadata.push(Token::new( + RntbdResponseToken::RequestDurationMilliseconds, + TokenValue::Double(value), + )); + } let metadata_len: usize = metadata.iter().map(Token::encoded_len).sum(); let header_len = u32::try_from(24 + metadata_len) @@ -195,6 +227,10 @@ impl RntbdResponse { let mut global_committed_lsn = None; let mut transport_request_id = None; let mut session_token = None; + let mut item_count = None; + let mut query_metrics = None; + let mut index_utilization = None; + let mut request_duration_ms = None; while !frame.is_empty() { let token = Token::read_from(&mut frame)?; @@ -238,6 +274,18 @@ impl RntbdResponse { Ok(RntbdResponseToken::SessionToken) => { session_token = Some(expect_string(token, "SessionToken")?); } + Ok(RntbdResponseToken::ItemCount) => { + item_count = Some(expect_u32(token, "ItemCount")?); + } + Ok(RntbdResponseToken::QueryMetrics) => { + query_metrics = Some(expect_string(token, "QueryMetrics")?); + } + Ok(RntbdResponseToken::IndexUtilization) => { + index_utilization = Some(expect_string(token, "IndexUtilization")?); + } + Ok(RntbdResponseToken::RequestDurationMilliseconds) => { + request_duration_ms = Some(expect_f64(token, "RequestDurationMilliseconds")?); + } Err(()) => {} } } @@ -271,6 +319,10 @@ impl RntbdResponse { global_committed_lsn, transport_request_id, session_token, + item_count, + query_metrics, + index_utilization, + request_duration_ms, }) } } @@ -325,6 +377,10 @@ mod tests { global_committed_lsn: Some(13), transport_request_id: Some(14), session_token: Some("1#12".to_owned()), + item_count: Some(15), + query_metrics: Some("totalExecutionTimeInMs=1.5".to_owned()), + index_utilization: Some("{}".to_owned()), + request_duration_ms: Some(2.5), }; let mut bytes = Vec::new(); response.write(&mut bytes).unwrap(); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs index ae166084d46..886f0b003bd 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/rntbd/tokens.rs @@ -671,6 +671,14 @@ pub(super) enum RntbdResponseToken { TransportRequestId, /// Session token. SessionToken, + /// Item count for feed-style (query/read-feed) responses. + ItemCount, + /// Serialized query execution metrics. + QueryMetrics, + /// Serialized index utilization/advice metrics. + IndexUtilization, + /// Backend request duration in milliseconds. + RequestDurationMilliseconds, } impl From for TokenId { @@ -681,14 +689,18 @@ impl From for TokenId { RntbdResponseToken::ETag => 0x0004, RntbdResponseToken::RetryAfterMilliseconds => 0x000C, RntbdResponseToken::Lsn => 0x0013, + RntbdResponseToken::ItemCount => 0x0014, RntbdResponseToken::RequestCharge => 0x0015, RntbdResponseToken::OwnerFullName => 0x0017, RntbdResponseToken::SubStatus => 0x001C, RntbdResponseToken::PartitionKeyRangeId => 0x0021, - RntbdResponseToken::ItemLsn => 0x0032, + RntbdResponseToken::QueryMetrics => 0x0028, RntbdResponseToken::GlobalCommittedLsn => 0x0029, + RntbdResponseToken::ItemLsn => 0x0032, RntbdResponseToken::TransportRequestId => 0x0035, RntbdResponseToken::SessionToken => 0x003E, + RntbdResponseToken::IndexUtilization => 0x0044, + RntbdResponseToken::RequestDurationMilliseconds => 0x0051, }) } } @@ -703,14 +715,18 @@ impl TryFrom for RntbdResponseToken { 0x0004 => Ok(Self::ETag), 0x000C => Ok(Self::RetryAfterMilliseconds), 0x0013 => Ok(Self::Lsn), + 0x0014 => Ok(Self::ItemCount), 0x0015 => Ok(Self::RequestCharge), 0x0017 => Ok(Self::OwnerFullName), 0x001C => Ok(Self::SubStatus), 0x0021 => Ok(Self::PartitionKeyRangeId), + 0x0028 => Ok(Self::QueryMetrics), 0x0032 => Ok(Self::ItemLsn), 0x0029 => Ok(Self::GlobalCommittedLsn), 0x0035 => Ok(Self::TransportRequestId), 0x003E => Ok(Self::SessionToken), + 0x0044 => Ok(Self::IndexUtilization), + 0x0051 => Ok(Self::RequestDurationMilliseconds), _ => Err(()), } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs index fc9b03cea3b..1bf486154a4 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/gateway_v2.rs @@ -494,23 +494,15 @@ async fn encode_response( global_committed_lsn: header_i64(headers, "x-ms-global-committed-lsn"), transport_request_id: header_u32(headers, "x-ms-transport-request-id"), session_token: header_string(headers, "x-ms-session-token"), + item_count: header_u32(headers, "x-ms-item-count"), + query_metrics: header_string(headers, "x-ms-documentdb-query-metrics"), + index_utilization: header_string(headers, "x-ms-cosmos-index-utilization"), + request_duration_ms: header_f64(headers, "x-ms-request-duration-ms"), }; let mut body = Vec::new(); rntbd.write(&mut body).map_err(gateway_v2_internal_error)?; let mut outer_headers = Headers::new(); outer_headers.insert("content-type", "application/octet-stream"); - for name in [ - "x-ms-request-duration-ms", - "lsn", - "x-ms-item-lsn", - "x-ms-global-committed-lsn", - "x-ms-documentdb-query-metrics", - "x-ms-cosmos-index-utilization", - ] { - if let Some(value) = header_string(headers, name) { - outer_headers.insert(name, value); - } - } Ok(AsyncRawResponse::from_bytes( status.status_code(), outer_headers, From 99498e09d64e0cd201d42f8e94123a1c78b7ecdb Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 23:03:03 +0000 Subject: [PATCH 39/44] Update validation.rs --- .../in_memory_emulator_tests/validation.rs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs index 2629cb48ae7..5a48662b8aa 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs @@ -233,21 +233,6 @@ impl HeaderValidationSpec { /// in-memory emulator cannot faithfully reproduce. Query comparison tests /// should log those values for diagnostics, while using this spec for basic /// header sanity and presence checks. - /// - /// `item_count` uses `Exists` rather than `Symmetric`: a real backend is - /// free to interleave empty probe pages (0 items, but a continuation) with - /// non-empty ones while scanning a partition/range, even when the query is - /// guaranteed to eventually yield a document -- e.g. R1 (0 docs, `CT1`), - /// R2 (2 docs, `CT2`), R3 (1 doc, no continuation). Such empty pages may - /// omit `x-ms-item-count` entirely. The in-memory emulator has no - /// equivalent scan-in-progress state and always emits exactly one item - /// per page, so page *count* can coincidentally match between backends - /// (e.g. 3 real pages vs. 3 emulator pages) while page *boundaries* don't - /// align at all. Since `compare_page_headers_if_aligned` zips pages by - /// index once counts match, a `Symmetric` rule ends up comparing - /// unrelated pages (e.g. empty real page 0 vs. non-empty emulator page 0) - /// and produces false positives (e.g. `hash_partition_scope` in - /// `query_comparison.rs`). pub fn for_query_operation() -> Self { Self::new() .with_rule("activity_id", HeaderMatch::Exists) @@ -255,7 +240,7 @@ impl HeaderValidationSpec { .with_rule("session_token", HeaderMatch::Exists) .with_rule("etag", HeaderMatch::Symmetric) .with_rule("continuation", HeaderMatch::Symmetric) - .with_rule("item_count", HeaderMatch::Exists) + .with_rule("item_count", HeaderMatch::Symmetric) .with_rule("index_metrics", HeaderMatch::Ignore) .with_rule("query_metrics", HeaderMatch::Ignore) .with_rule("server_duration_ms", HeaderMatch::Exists) From 95eda00b6318703b9debacec8b59eddb11990835 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 17 Jul 2026 23:11:55 +0000 Subject: [PATCH 40/44] Fix accidental yml changes --- .../templates/stages/archetype-sdk-client.yml | 4 ---- sdk/cosmos/ci.yml | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 28672aa05dd..8167c3d2cbf 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -55,9 +55,6 @@ parameters: - name: AdditionalMatrixConfigs type: object default: [] -- name: BuildOnlyAdditionalMatrixConfigs - type: object - default: [] - name: MatrixFilters type: object default: [] @@ -125,7 +122,6 @@ extends: MatrixConfigs: - ${{ parameters.MatrixConfigs }} - ${{ parameters.AdditionalMatrixConfigs }} - - ${{ parameters.BuildOnlyAdditionalMatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index 5307b1303ef..62d1404a4d7 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -61,14 +61,11 @@ extends: Selection: sparse NonSparseParameters: RustToolchainName GenerateVMJobs: true - LiveTestMatrixConfigs: - - Name: Cosmos_live_test - Path: sdk/cosmos/live-platform-matrix.json - Selection: sparse - GenerateVMJobs: true - # Emulator jobs run only in Build. They do not enter live stages, deploy - # Azure resources, or replace live-test credentials with local endpoints. - BuildOnlyAdditionalMatrixConfigs: + # Emulator jobs run only in Build. `MatrixConfigs` (unlike + # `AdditionalMatrixConfigs`) is never consumed by the live-test stage in + # archetype-sdk-client.yml, so these entries don't enter live stages, + # deploy Azure resources, or replace live-test credentials with local + # endpoints. - Name: Cosmos_vnext_emulator Path: sdk/cosmos/vnext-emulator-matrix.json Selection: all @@ -79,3 +76,9 @@ extends: Path: sdk/cosmos/inmemory-emulator-matrix.json Selection: all GenerateVMJobs: true + LiveTestMatrixConfigs: + - Name: Cosmos_live_test + Path: sdk/cosmos/live-platform-matrix.json + Selection: sparse + GenerateVMJobs: true + From 33b985e6259b9529a00ca86cf4e804368ca1f67c Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 18 Jul 2026 09:38:07 +0000 Subject: [PATCH 41/44] Update gateway_v2_dispatch.rs --- .../src/driver/transport/gateway_v2_dispatch.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs index 7630b669fb6..c6509a468d2 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/transport/gateway_v2_dispatch.rs @@ -468,15 +468,15 @@ pub(crate) fn unwrap_response_for_gateway_v2( if let Some(retry_after_ms) = response.retry_after_ms { headers.insert("x-ms-retry-after-ms", retry_after_ms.to_string()); } - if let Some(lsn) = response.lsn.filter(|value| *value != 0) { + if let Some(lsn) = response.lsn { let value = lsn.to_string(); headers.insert(response_header_names::LSN, value.clone()); headers.insert(X_MS_LSN, value); } - if let Some(item_lsn) = response.item_lsn.filter(|value| *value != 0) { + if let Some(item_lsn) = response.item_lsn { headers.insert(response_header_names::ITEM_LSN, item_lsn.to_string()); } - if let Some(global_committed_lsn) = response.global_committed_lsn.filter(|value| *value != 0) { + if let Some(global_committed_lsn) = response.global_committed_lsn { headers.insert(X_MS_GLOBAL_COMMITTED_LSN, global_committed_lsn.to_string()); } if let Some(partition_key_range_id) = response.partition_key_range_id { From ce144b2560e8616cd6f330108fa73b20a71e203c Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 18 Jul 2026 09:59:55 +0000 Subject: [PATCH 42/44] Update dual_backend.rs --- .../in_memory_emulator_tests/dual_backend.rs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/dual_backend.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/dual_backend.rs index 875c7dda37c..20788d04e79 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/dual_backend.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/dual_backend.rs @@ -200,13 +200,24 @@ impl DualBackend { } /// Creates a database on the real account via the driver. + /// + /// Tolerates a 409 Conflict from the create itself: `create_database` is + /// not idempotent, but a client-side timeout (e.g. `TransportGenerated503` + /// after the request timeout elapses) doesn't mean the server never + /// processed the request — a retry can legitimately observe the database + /// already exists. Treating that as success avoids spurious test failures + /// from slow/flaky network conditions against the real account. pub async fn create_real_database(&self, db_name: &str) -> Result<(), Box> { if let (Some(driver), Some(account)) = (&self.real_driver, &self.real_account) { let body = serde_json::to_vec(&serde_json::json!({"id": db_name}))?; let op = CosmosOperation::create_database(account.clone()).with_body(body); let result = driver .execute_singleton_operation(op, OperationOptions::default()) - .await?; + .await; + let result = match result { + Err(error) if error.status().status_code() == StatusCode::Conflict => return Ok(()), + other => other?, + }; assert!( result.status().is_success(), "Failed to create real database '{}': {}", @@ -256,7 +267,15 @@ impl DualBackend { let op = CosmosOperation::create_container(db_ref).with_body(body); let result = driver .execute_singleton_operation(op, OperationOptions::default()) - .await?; + .await; + // See `create_real_database` for why a 409 Conflict here is + // tolerated rather than propagated: a client-side timeout on the + // initial create attempt doesn't preclude the server having + // already processed it. + let result = match result { + Err(error) if error.status().status_code() == StatusCode::Conflict => return Ok(()), + other => other?, + }; assert!( result.status().is_success(), "Failed to create real container '{}/{}': {}", From e59c8a1ebf5bb884c115347ea4cf3c2fb52b9c07 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 18 Jul 2026 10:07:40 +0000 Subject: [PATCH 43/44] Update test_client.rs --- .../tests/framework/test_client.rs | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/framework/test_client.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/framework/test_client.rs index 575d0da1253..1aa660f5449 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/framework/test_client.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/framework/test_client.rs @@ -632,6 +632,13 @@ impl DriverTestRunContext { } /// Creates a database using the driver. + /// + /// Tolerates a 409 Conflict from the create itself: `create_database` is + /// not idempotent, but a client-side timeout (surfaced as a synthetic + /// `TransportGenerated503`) doesn't mean the server never processed the + /// request — the driver's own region-failover retry can legitimately + /// observe the database already exists. Treating that as success avoids + /// spurious test failures from slow/flaky network conditions. pub async fn create_database( &self, db_name: &str, @@ -648,7 +655,16 @@ impl DriverTestRunContext { let result = driver .execute_singleton_operation(operation, OperationOptions::default()) - .await?; + .await; + let result = match result { + Err(error) if error.status().status_code() == StatusCode::Conflict => { + return Ok(DatabaseReference::from_name( + self.client.account.clone(), + db_name.to_string(), + )); + } + other => other?, + }; // Check for success status (201 Created) let diagnostics = result.diagnostics(); @@ -743,15 +759,27 @@ impl DriverTestRunContext { let operation = CosmosOperation::create_container(database.clone()).with_body(body.into_bytes()); - let result = driver + let create_result = driver .execute_singleton_operation(operation, OperationOptions::default()) - .await?; - - // Check for success status (201 Created) - let diagnostics = result.diagnostics(); - let status = diagnostics.status(); - if !status.map(|s| s.is_success()).unwrap_or(false) { - return Err(format!("Failed to create container, status: {:?}", status).into()); + .await; + // Tolerate a 409 Conflict from the create itself: a client-side + // timeout (surfaced as a synthetic `TransportGenerated503`) doesn't + // mean the server never processed the request — the driver's own + // region-failover retry can legitimately observe the container + // already exists. Fall through to the resolve-retry loop below + // exactly as a successful create would, since that's what actually + // produces the `ContainerReference` this method returns. + match create_result { + Err(error) if error.status().status_code() == StatusCode::Conflict => {} + other => { + let result = other?; + // Check for success status (201 Created) + let diagnostics = result.diagnostics(); + let status = diagnostics.status(); + if !status.map(|s| s.is_success()).unwrap_or(false) { + return Err(format!("Failed to create container, status: {:?}", status).into()); + } + } } let db_name = database .name() From 32cc9bc40db5533e9f9608e2809902ccc461623c Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 20 Jul 2026 08:16:12 +0000 Subject: [PATCH 44/44] Update validation.rs --- .../tests/in_memory_emulator_tests/validation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs index 5a48662b8aa..4a2cf639160 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/validation.rs @@ -98,7 +98,7 @@ impl HeaderValidationSpec { /// metrics (no-op on point ops). /// - **`RequiredInEmulatorOnly`** — emulator must emit this diagnostic /// header, but Gateway-backed real/emulator responses may omit it. - /// Used for `internal_partition_id` on point operations. + /// Used for `internal_partition_id` on point and query operations. /// - **`Ignore`** — emulator does not (and intentionally will not) /// produce these headers, or they encode internal state that has no /// public meaning for the operation under test: `item_local_lsn`, @@ -263,7 +263,7 @@ impl HeaderValidationSpec { .with_rule("resource_usage", HeaderMatch::Ignore) .with_rule("has_tentative_writes", HeaderMatch::Symmetric) .with_rule("partition_key_range_id", HeaderMatch::Symmetric) - .with_rule("internal_partition_id", HeaderMatch::Symmetric) + .with_rule("internal_partition_id", HeaderMatch::RequiredInEmulatorOnly) .with_rule("log_results", HeaderMatch::Symmetric) .with_rule( "collection_index_transformation_progress",