chore: introduce request retries

ic-agent/src/agent/http_transport/reqwest_transport.rs

@@ -28,6 +28,7 @@ pub struct ReqwestTransport {
     route_provider: Arc<dyn RouteProvider>,
     client: Client,
     max_response_body_size: Option<usize>,
+    max_tcp_error_retries: usize,
 }
 
 #[doc(hidden)]
@@ -68,6 +69,7 @@ impl ReqwestTransport {
             route_provider,
             client,
             max_response_body_size: None,
+            max_tcp_error_retries: 0,
         })
     }
 
@@ -79,15 +81,49 @@ impl ReqwestTransport {
         }
     }
 
+    /// Sets a max number of retries for tcp connection errors.
+    pub fn with_max_tcp_errors_retries(self, retries: usize) -> Self {
+        ReqwestTransport {
+            max_tcp_error_retries: retries,
+            ..self
+        }
+    }
+
     async fn request(
         &self,
-        http_request: Request,
+        method: Method,
+        endpoint: &str,
+        body: Option<Vec<u8>>,
     ) -> Result<(StatusCode, HeaderMap, Vec<u8>), AgentError> {
-        let response = self
-            .client
-            .execute(http_request)
-            .await
-            .map_err(|x| AgentError::TransportError(Box::new(x)))?;
+        let mut retry_count = 0;
+
+        let response = loop {
+            // RouteProvider generates urls dynamically. Some of these urls can be potentially unhealthy.
+            // TCP related errors (host unreachable, connection refused, connection timed out, connection reset) can be safely retried with a newly generated url.
+            let url = self.route_provider.route()?.join(endpoint)?;
+
+            let mut http_request = Request::new(method.clone(), url);
+
+            http_request
+                .headers_mut()
+                .insert(CONTENT_TYPE, "application/cbor".parse().unwrap());
+
+            *http_request.body_mut() = body.as_ref().cloned().map(Body::from);
+
+            match self.client.execute(http_request).await {
+                Ok(response) => break response,
+                Err(err) => {
+                    if err.is_connect() {
+                        if retry_count >= self.max_tcp_error_retries {
+                            return Err(AgentError::TransportError(Box::new(err)));
+                        }
+                        retry_count += 1;
+                        continue;
+                    }
+                    return Err(AgentError::TransportError(Box::new(err)));
+                }
+            }
+        };
 
         let http_status = response.status();
         let response_headers = response.headers().clone();
@@ -128,16 +164,10 @@ impl ReqwestTransport {
         endpoint: &str,
         body: Option<Vec<u8>>,
     ) -> Result<Vec<u8>, AgentError> {
-        let url = self.route_provider.route()?.join(endpoint)?;
-        let mut http_request = Request::new(method, url);
-        http_request
-            .headers_mut()
-            .insert(CONTENT_TYPE, "application/cbor".parse().unwrap());
-
-        *http_request.body_mut() = body.map(Body::from);
-
         let request_result = loop {
-            let result = self.request(http_request.try_clone().unwrap()).await?;
+            let result = self
+                .request(method.clone(), endpoint, body.as_ref().cloned())
+                .await?;
             if result.0 != StatusCode::TOO_MANY_REQUESTS {
                 break result;
             }