feat(grpc): broaden retry policy to RESOURCE_EXHAUSTED and ABORTED

Rename `retry_on_unavailable` → `retry_on_transient` in `rust/src/retry.rs`
and add a configurable `retryable_codes: HashSet<i32>` field to `RetryConfig`.

Default retry set is now {UNAVAILABLE, RESOURCE_EXHAUSTED, ABORTED}:
- RESOURCE_EXHAUSTED maps to HTTP 429 (rate limit) — should retry with backoff
- ABORTED signals an optimistic-concurrency conflict — tonic docs recommend retry
- DEADLINE_EXCEEDED is intentionally excluded — retrying without resetting the
  deadline just fails again immediately

All 12 transport.rs callsites updated to `retry_on_transient`. New/updated
tests cover the expanded code set and a custom-codes override path.

Co-authored-by: P-0228

Author: jhamon

AGENTS.md

@@ -283,7 +283,9 @@ except PineconeError as e:
     print(e)  # Validation, timeout, or connection error
 ```
 
-**Retry behavior:** All HTTP methods (GET, HEAD, POST, PUT, PATCH, DELETE) are automatically retried on transient failures: connection errors (`httpx.TransportError`), 408 Request Timeout, 429 Too Many Requests (honoring `Retry-After`), and 5xx (500, 502, 503, 504). Pinecone's data-plane writes are idempotent at the server (upsert overwrites by ID, delete-by-ID is idempotent, update-by-ID is idempotent), so retrying upsert/query/fetch/delete/update on transient errors is safe. Backoff is floored full jitter: `uniform(0.1 * base, base)` where `base = min(backoff_factor**attempt, max_wait)`. Configure via `RetryConfig`.
+**Retry behavior (HTTP):** All HTTP methods (GET, HEAD, POST, PUT, PATCH, DELETE) are automatically retried on transient failures: connection errors (`httpx.TransportError`), 408 Request Timeout, 429 Too Many Requests (honoring `Retry-After`), and 5xx (500, 502, 503, 504). Pinecone's data-plane writes are idempotent at the server (upsert overwrites by ID, delete-by-ID is idempotent, update-by-ID is idempotent), so retrying upsert/query/fetch/delete/update on transient errors is safe. Backoff is floored full jitter: `uniform(0.1 * base, base)` where `base = min(backoff_factor**attempt, max_wait)`. Configure via `RetryConfig`.
+
+**Retry behavior (gRPC):** gRPC retries on UNAVAILABLE, RESOURCE_EXHAUSTED (rate limit), and ABORTED (concurrency conflict). DEADLINE_EXCEEDED is not retried — set a longer client timeout instead. All three default retryable codes are safe for Pinecone data-plane operations (upsert, query, fetch, delete-by-id, update), which are idempotent. Backoff uses full-jitter exponential: `uniform(0, min(max_backoff, initial_backoff * multiplier^attempt))`. The set of retryable codes is configurable via `RetryConfig.retryable_codes`.
 
 ## Response Objects
 

rust/src/retry.rs

@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::future::Future;
 use std::time::Duration;
 
@@ -15,6 +16,10 @@ pub struct RetryConfig {
     pub max_backoff: Duration,
     /// Backoff multiplier applied each attempt.
     pub multiplier: u32,
+    /// gRPC status codes that trigger a retry. Defaults to UNAVAILABLE, RESOURCE_EXHAUSTED,
+    /// ABORTED — Pinecone data-plane operations (upsert, query, fetch, delete-by-id, update)
+    /// are idempotent and safe to retry on these transient codes.
+    pub retryable_codes: HashSet<i32>,
 }
 
 impl Default for RetryConfig {
@@ -24,18 +29,25 @@ impl Default for RetryConfig {
             initial_backoff: Duration::from_millis(100),
             max_backoff: Duration::from_millis(1600),
             multiplier: 2,
+            retryable_codes: [
+                tonic::Code::Unavailable as i32,
+                tonic::Code::ResourceExhausted as i32,
+                tonic::Code::Aborted as i32,
+            ]
+            .into_iter()
+            .collect(),
         }
     }
 }
 
-/// Execute an async gRPC operation with retry on `Code::Unavailable`.
+/// Execute an async gRPC operation with retry on transient error codes.
 ///
 /// Uses full-jitter exponential backoff:
 ///   delay = random(0, min(max_backoff, initial_backoff * multiplier^attempt))
 ///
-/// Only retries on `tonic::Code::Unavailable`. All other error codes are
-/// returned immediately without retry.
-pub async fn retry_on_unavailable<F, Fut, T>(
+/// Retries on any code listed in `config.retryable_codes` (default: UNAVAILABLE,
+/// RESOURCE_EXHAUSTED, ABORTED). All other error codes are returned immediately without retry.
+pub async fn retry_on_transient<F, Fut, T>(
     config: &RetryConfig,
     mut operation: F,
 ) -> Result<T, Status>
@@ -48,7 +60,7 @@ where
     loop {
         match operation().await {
             Ok(val) => return Ok(val),
-            Err(status) if status.code() == tonic::Code::Unavailable => {
+            Err(status) if config.retryable_codes.contains(&(status.code() as i32)) => {
                 if attempt >= config.max_retries {
                     return Err(status);
                 }
@@ -62,6 +74,8 @@ where
                     let ms = rand::rng().random_range(0..=capped.as_millis() as u64);
                     Duration::from_millis(ms)
                 };
+                // TODO: honor server-supplied pushback hints (grpc-retry-pushback-ms trailer
+                // or Retry-After in trailers) before falling back to computed jitter backoff.
                 tokio::time::sleep(jittered).await;
                 attempt += 1;
             }
@@ -82,6 +96,7 @@ mod tests {
             initial_backoff: Duration::from_millis(1), // fast for tests
             max_backoff: Duration::from_millis(10),
             multiplier: 2,
+            ..Default::default()
         }
     }
 
@@ -91,7 +106,7 @@ mod tests {
         let count = call_count.clone();
 
         let config = test_config(5);
-        let result = retry_on_unavailable(&config, || {
+        let result = retry_on_transient(&config, || {
             let count = count.clone();
             async move {
                 let n = count.fetch_add(1, Ordering::SeqCst);
@@ -116,7 +131,7 @@ mod tests {
         let count = call_count.clone();
 
         let config = test_config(5);
-        let result = retry_on_unavailable(&config, || {
+        let result = retry_on_transient(&config, || {
             let count = count.clone();
             async move {
                 count.fetch_add(1, Ordering::SeqCst);
@@ -132,32 +147,110 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn no_retry_on_resource_exhausted() {
+    async fn retry_occurs_on_resource_exhausted() {
         let call_count = Arc::new(AtomicU32::new(0));
         let count = call_count.clone();
 
         let config = test_config(5);
-        let result = retry_on_unavailable(&config, || {
+        let result = retry_on_transient(&config, || {
+            let count = count.clone();
+            async move {
+                let n = count.fetch_add(1, Ordering::SeqCst);
+                if n < 2 {
+                    Err(Status::resource_exhausted("rate limited"))
+                } else {
+                    Ok::<(), Status>(())
+                }
+            }
+        })
+        .await;
+
+        assert!(result.is_ok());
+        // Initial call + 2 retries = 3 total calls
+        assert_eq!(call_count.load(Ordering::SeqCst), 3);
+    }
+
+    #[tokio::test]
+    async fn retry_occurs_on_aborted() {
+        let call_count = Arc::new(AtomicU32::new(0));
+        let count = call_count.clone();
+
+        let config = test_config(5);
+        let result = retry_on_transient(&config, || {
+            let count = count.clone();
+            async move {
+                let n = count.fetch_add(1, Ordering::SeqCst);
+                if n < 2 {
+                    Err(Status::aborted("conflict"))
+                } else {
+                    Ok::<(), Status>(())
+                }
+            }
+        })
+        .await;
+
+        assert!(result.is_ok());
+        // Initial call + 2 retries = 3 total calls
+        assert_eq!(call_count.load(Ordering::SeqCst), 3);
+    }
+
+    #[tokio::test]
+    async fn no_retry_on_internal() {
+        let call_count = Arc::new(AtomicU32::new(0));
+        let count = call_count.clone();
+
+        let config = test_config(5);
+        let result = retry_on_transient(&config, || {
             let count = count.clone();
             async move {
                 count.fetch_add(1, Ordering::SeqCst);
-                Err::<(), Status>(Status::resource_exhausted("rate limited"))
+                Err::<(), Status>(Status::internal("oops"))
             }
         })
         .await;
 
         assert!(result.is_err());
-        assert_eq!(result.unwrap_err().code(), tonic::Code::ResourceExhausted);
+        assert_eq!(result.unwrap_err().code(), tonic::Code::Internal);
         assert_eq!(call_count.load(Ordering::SeqCst), 1);
     }
 
+    #[tokio::test]
+    async fn custom_retryable_codes_override() {
+        let call_count = Arc::new(AtomicU32::new(0));
+        let count = call_count.clone();
+
+        let config = RetryConfig {
+            max_retries: 3,
+            initial_backoff: Duration::from_millis(1),
+            max_backoff: Duration::from_millis(10),
+            multiplier: 2,
+            retryable_codes: HashSet::from([tonic::Code::DeadlineExceeded as i32]),
+        };
+        let result = retry_on_transient(&config, || {
+            let count = count.clone();
+            async move {
+                let n = count.fetch_add(1, Ordering::SeqCst);
+                if n < 2 {
+                    Err(Status::deadline_exceeded("timeout"))
+                } else {
+                    Ok::<(), Status>(())
+                }
+            }
+        })
+        .await;
+
+        assert!(result.is_ok());
+        // DEADLINE_EXCEEDED is retried under this custom config
+        assert_eq!(call_count.load(Ordering::SeqCst), 3);
+    }
+
     #[tokio::test]
     async fn respects_max_retry_count() {
         let call_count = Arc::new(AtomicU32::new(0));
         let count = call_count.clone();
 
         let config = test_config(3);
-        let result = retry_on_unavailable(&config, || {
+        let result = retry_on_transient(&config, || {
             let count = count.clone();
             async move {
                 count.fetch_add(1, Ordering::SeqCst);
@@ -178,7 +271,7 @@ mod tests {
         let count = call_count.clone();
 
         let config = test_config(0);
-        let result = retry_on_unavailable(&config, || {
+        let result = retry_on_transient(&config, || {
             let count = count.clone();
             async move {
                 count.fetch_add(1, Ordering::SeqCst);
@@ -200,23 +293,25 @@ mod tests {
             initial_backoff: Duration::from_millis(10),
             max_backoff: Duration::from_millis(500),
             multiplier: 2,
+            ..Default::default()
         };
         let config_3 = RetryConfig {
             max_retries: 3,
             initial_backoff: Duration::from_millis(10),
             max_backoff: Duration::from_millis(500),
             multiplier: 2,
+            ..Default::default()
         };
 
         let start_1 = std::time::Instant::now();
-        let _ = retry_on_unavailable(&config_1, || async {
+        let _ = retry_on_transient(&config_1, || async {
             Err::<(), Status>(Status::unavailable("unavailable"))
         })
         .await;
         let elapsed_1 = start_1.elapsed();
 
         let start_3 = std::time::Instant::now();
-        let _ = retry_on_unavailable(&config_3, || async {
+        let _ = retry_on_transient(&config_3, || async {
             Err::<(), Status>(Status::unavailable("unavailable"))
         })
         .await;
@@ -233,7 +328,7 @@ mod tests {
     async fn success_on_first_attempt_returns_immediately() {
         let config = test_config(5);
         let result =
-            retry_on_unavailable(&config, || async { Ok::<&str, Status>("immediate") }).await;
+            retry_on_transient(&config, || async { Ok::<&str, Status>("immediate") }).await;
 
         assert!(result.is_ok());
         assert_eq!(result.unwrap(), "immediate");