Dynamic default part size

CMakeLists.txt

@@ -108,6 +108,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake"
 
 include(CTest)
 if (BUILD_TESTING)
+    add_definitions(-DAWS_C_S3_ENABLE_TEST_STUBS)
     add_subdirectory(tests)
     if (NOT BYO_CRYPTO AND NOT CMAKE_CROSSCOMPILING)
         add_subdirectory(samples)

include/aws/s3/private/s3_auto_ranged_get.h

@@ -21,6 +21,12 @@ struct aws_s3_auto_ranged_get {
 
     struct aws_string *etag;
 
+    /* Estimated object stored part size based on ETag analysis */
+    uint64_t estimated_object_stored_part_size;
+    /* Part size was set or not from user for this meta request. */
+    bool part_size_set;
+    bool force_dynamic_part_size;
+
     bool initial_message_has_start_range;
     bool initial_message_has_end_range;
     uint64_t initial_range_start;
@@ -74,6 +80,7 @@ AWS_S3_API struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
     struct aws_allocator *allocator,
     struct aws_s3_client *client,
     size_t part_size,
+    bool part_size_set,
     const struct aws_s3_meta_request_options *options);
 
 AWS_EXTERN_C_END

include/aws/s3/private/s3_client_impl.h

@@ -178,7 +178,10 @@ struct aws_s3_client_vtable {
         struct aws_http_connection *client_connection,
         const struct aws_http_make_request_options *options);
 
-    void (*after_prepare_upload_part_finish)(struct aws_s3_request *request, struct aws_http_message *message);
+#ifdef AWS_C_S3_ENABLE_TEST_STUBS
+    /********************* TEST ONLY STUB **************************/
+    void (*after_prepare_upload_part_finish_stub)(struct aws_s3_request *request, struct aws_http_message *message);
+#endif
 };
 
 struct aws_s3_upload_part_timeout_stats {
@@ -234,10 +237,17 @@ struct aws_s3_client {
      * to meta requests for use. */
     const size_t part_size;
 
+    bool part_size_set;
+
     /* Size of parts for files when doing gets or puts.  This exists on the client as configurable option that is passed
      * to meta requests for use. */
     const uint64_t max_part_size;
 
+    /* Calculated optimal range size for GET operations based on client configuration (memory limits, throughput
+     * targets). This is used when part_size is not explicitly configured, replacing the default with reasonable
+     * calculation. Value is calculated during client initialization and remains constant for the client's lifetime. */
+    const uint64_t optimal_range_size;
+
     /* File I/O options. */
     bool fio_options_set;
     struct aws_s3_file_io_options fio_opts;

include/aws/s3/private/s3_meta_request_impl.h

@@ -122,6 +122,12 @@ struct aws_s3_meta_request_vtable {
 
     /* Pause the given request */
     int (*pause)(struct aws_s3_meta_request *meta_request, struct aws_s3_meta_request_resume_token **resume_token);
+
+#ifdef AWS_C_S3_ENABLE_TEST_STUBS
+    /********************* TEST ONLY STUB **************************/
+    /* A stub to the update implementation from meta request with the lock held. Only for tests. */
+    bool (*synced_update_stub)(struct aws_s3_meta_request *meta_request);
+#endif
 };
 
 /**
@@ -407,9 +413,11 @@ void aws_s3_meta_request_add_event_for_delivery_synced(
 bool aws_s3_meta_request_are_events_out_for_delivery_synced(struct aws_s3_meta_request *meta_request);
 
 /* Cancel the requests with cancellable HTTP stream for the meta request */
+AWS_S3_API
 void aws_s3_meta_request_cancel_cancellable_requests_synced(struct aws_s3_meta_request *meta_request, int error_code);
 
 /* Cancel the pending buffer futures for the meta request */
+AWS_S3_API
 void aws_s3_meta_request_cancel_pending_buffer_futures_synced(struct aws_s3_meta_request *meta_request, int error_code);
 
 /* Asynchronously read from the meta request's input stream. Should always be done outside of any mutex,

include/aws/s3/private/s3_request.h

@@ -122,8 +122,19 @@ struct aws_s3_request_metrics {
         int error_code;
         /* Retry attempt. */
         uint32_t retry_attempt;
+        /* Is the memory for the request allocated from the buffer pool or not. */
+        bool memory_allocated_from_pool;
     } crt_info_metrics;
 
+    struct {
+        /* Beginning range of this part. */
+        uint64_t part_range_start;
+        /* Last byte of this part. */
+        uint64_t part_range_end;
+        /* Part number that this request refers to. */
+        uint32_t part_number;
+    } part_info_metrics;
+
     struct aws_ref_count ref_count;
 };
 

include/aws/s3/private/s3_util.h

@@ -160,6 +160,15 @@ extern const double g_default_throughput_target_gbps;
 
 AWS_S3_API
 extern const uint64_t g_streaming_object_size_threshold;
+
+AWS_S3_API
+extern const uint64_t g_default_part_size_fallback;
+
+AWS_S3_API
+extern const uint64_t g_default_max_part_size;
+
+AWS_S3_API
+extern const uint64_t g_s3_optimal_range_size_alignment;
 /**
  * Returns AWS_S3_REQUEST_TYPE_UNKNOWN if name doesn't map to an enum value.
  */
@@ -318,6 +327,51 @@ int aws_s3_check_headers_for_checksum(
     struct aws_byte_buf *out_checksum_buffer,
     bool meta_request_level);
 
+/**
+ * Calculate client-level optimal range size based on memory and connection constraints.
+ * This function is called during client initialization to determine the base range size
+ * using the formula: MemoryLimit / concurrency / divisor.
+ * The result is rounded up to ensure proper alignment and applies minimum size constraints.
+ *
+ * @param memory_limit_in_bytes Total memory limit available for buffering
+ * @param max_connections Maximum number of concurrent connections
+ * @param out_client_optimal_range_size Output parameter for calculated client-level optimal range size
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (caller should fall back to default)
+ */
+AWS_S3_API
+int aws_s3_calculate_client_optimal_range_size(
+    uint64_t memory_limit_in_bytes,
+    uint32_t max_connections,
+    uint64_t *out_client_optimal_range_size);
+
+/**
+ * Calculate request-level optimal range size by considering object-specific information.
+ * This function is called per request to adjust the client-level range size based on
+ * estimated object stored part size using: min(client_optimal_range_size, estimated_object_stored_part_size).
+ *
+ * @param client_optimal_range_size The client-level optimal range size from initialization
+ * @param estimated_object_stored_part_size Estimated size of object stored parts in S3
+ * @param out_request_optimal_range_size Output parameter for calculated request-level optimal range size
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (caller should fall back to client size)
+ */
+AWS_S3_API
+int aws_s3_calculate_request_optimal_range_size(
+    uint64_t client_optimal_range_size,
+    uint64_t estimated_object_stored_part_size,
+    uint64_t *out_request_optimal_range_size);
+
+/**
+ * Extract the number of parts from an S3 ETag header value.
+ * S3 multipart upload ETags have the format "<hash>-<number_of_parts>".
+ * Single-part uploads have ETags without dashes.
+ *
+ * @param etag_header_value The ETag header value (may include quotes)
+ * @param out_num_parts Output parameter for the number of parts (1 for single-part uploads)
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (invalid ETag format)
+ */
+AWS_S3_API
+int aws_s3_extract_parts_from_etag(struct aws_byte_cursor etag_header_value, uint32_t *out_num_parts);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_UTIL_H */

include/aws/s3/s3_client.h

@@ -510,7 +510,9 @@ struct aws_s3_client_config {
      * Optional.
      * Size of parts the object will be downloaded or uploaded in, in bytes.
      * This only affects AWS_S3_META_REQUEST_TYPE_GET_OBJECT and AWS_S3_META_REQUEST_TYPE_PUT_OBJECT.
-     * If not set, this defaults to 8 MiB.
+     *
+     * If not set, a dynamic default part size will be used based on the throughput target, memory_limit_in_bytes.
+     *
      * The client will adjust the part size for AWS_S3_META_REQUEST_TYPE_PUT_OBJECT if needed for service limits (max
      * number of parts per upload is 10,000, minimum upload part size is 5 MiB).
      *
@@ -875,13 +877,22 @@ struct aws_s3_meta_request_options {
      * Optional.
      * Size of parts the object will be downloaded or uploaded in, in bytes.
      * This only affects AWS_S3_META_REQUEST_TYPE_GET_OBJECT and AWS_S3_META_REQUEST_TYPE_PUT_OBJECT.
-     * If not set, the value from `aws_s3_client_config.part_size` is used, which defaults to 8MiB.
+     *
+     * If not set, the value from `aws_s3_client_config.part_size` is used, which defaults to a dynamic value based on
+     * the throughput target, memory_limit_in_bytes and the requested object size.
      *
      * The client will adjust the part size for AWS_S3_META_REQUEST_TYPE_PUT_OBJECT if needed for service limits (max
      * number of parts per upload is 10,000, minimum upload part size is 5 MiB).
      */
     uint64_t part_size;
 
+    /**
+     * Optional - EXPERIMENTAL/UNSTABLE
+     * Set this to prefer for the dynamic default part_size over the part size set for both client and meta request for
+     * the best performance under the memory constrain, especially for getting large objects.
+     */
+    bool force_dynamic_part_size;
+
     /**
      * Optional.
      * The size threshold in bytes for when to use multipart uploads.
@@ -1604,6 +1615,26 @@ int aws_s3_request_metrics_get_error_code(const struct aws_s3_request_metrics *m
 AWS_S3_API
 uint32_t aws_s3_request_metrics_get_retry_attempt(const struct aws_s3_request_metrics *metrics);
 
+/* Get the memory for the request allocated from the pool or not. Cannot fail */
+AWS_S3_API
+bool aws_s3_request_metrics_get_memory_allocated_from_pool(const struct aws_s3_request_metrics *metrics);
+
+/* Get the beginning range of this part from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_range_start(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_part_range_start);
+
+/* Get the last byte of this part from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_range_end(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_part_range_end);
+
+/* Get the part number from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_number(const struct aws_s3_request_metrics *metrics, uint32_t *out_part_number);
+
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL
 

source/s3_auto_ranged_get.c

@@ -62,6 +62,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
     struct aws_allocator *allocator,
     struct aws_s3_client *client,
     size_t part_size,
+    bool part_size_set,
     const struct aws_s3_meta_request_options *options) {
     AWS_PRECONDITION(allocator);
     AWS_PRECONDITION(client);
@@ -91,6 +92,8 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
         return NULL;
     }
 
+    auto_ranged_get->part_size_set = part_size_set;
+    auto_ranged_get->force_dynamic_part_size = options->force_dynamic_part_size;
     struct aws_http_headers *headers = aws_http_message_get_headers(auto_ranged_get->base.initial_request_message);
     AWS_ASSERT(headers != NULL);
 
@@ -110,6 +113,7 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
         }
     }
     auto_ranged_get->initial_message_has_if_match_header = aws_http_headers_has(headers, g_if_match_header_name);
+
     auto_ranged_get->synced_data.first_part_size = auto_ranged_get->base.part_size;
     if (options->object_size_hint != NULL) {
         auto_ranged_get->object_size_hint_available = true;
@@ -200,6 +204,13 @@ static bool s_s3_auto_ranged_get_update(
     {
         aws_s3_meta_request_lock_synced_data(meta_request);
 
+#ifdef AWS_C_S3_ENABLE_TEST_STUBS
+        if (meta_request->vtable->synced_update_stub && meta_request->vtable->synced_update_stub(meta_request)) {
+            /* TEST ONLY, allow test to stub here. */
+            aws_s3_meta_request_unlock_synced_data(meta_request);
+            return true;
+        }
+#endif
         /* If nothing has set the "finish result" then this meta request is still in progress, and we can potentially
          * send additional requests. */
         if (!aws_s3_meta_request_has_finish_result_synced(meta_request)) {
@@ -319,8 +330,12 @@ static bool s_s3_auto_ranged_get_update(
                      * we could end up stuck in a situation where the user is
                      * waiting for more bytes before they'll open the window,
                      * and this implementation is waiting for more window before it will send more parts. */
-                    uint64_t read_data_requested =
-                        auto_ranged_get->synced_data.num_parts_requested * meta_request->part_size;
+                    uint64_t read_data_requested = 0;
+                    if (auto_ranged_get->synced_data.num_parts_requested > 0) {
+                        read_data_requested =
+                            (auto_ranged_get->synced_data.num_parts_requested - 1) * meta_request->part_size +
+                            auto_ranged_get->synced_data.first_part_size;
+                    }
                     if (read_data_requested >= meta_request->synced_data.read_window_running_total) {
 
                         /* Avoid spamming users with this DEBUG message */
@@ -757,29 +772,74 @@ static void s_s3_auto_ranged_get_request_finished(
 
             goto update_synced_data;
         }
-        if ((!request_failed || first_part_size_mismatch) && !auto_ranged_get->initial_message_has_if_match_header) {
-            AWS_ASSERT(auto_ranged_get->etag == NULL);
+        /* Always extract ETag header for part size estimation */
+        if (!request_failed || first_part_size_mismatch) {
             struct aws_byte_cursor etag_header_value;
+            AWS_ASSERT(auto_ranged_get->etag == NULL);
+            if (aws_http_headers_get(request->send_data.response_headers, g_etag_header_name, &etag_header_value) ==
+                AWS_OP_SUCCESS) {
+                AWS_LOGF_TRACE(
+                    AWS_LS_S3_META_REQUEST,
+                    "id=%p ETag received for the meta request. value is: " PRInSTR "",
+                    (void *)meta_request,
+                    AWS_BYTE_CURSOR_PRI(etag_header_value));
 
-            if (aws_http_headers_get(request->send_data.response_headers, g_etag_header_name, &etag_header_value)) {
+                if (!auto_ranged_get->initial_message_has_if_match_header) {
+                    /* Store ETag if needed for If-Match header */
+                    auto_ranged_get->etag =
+                        aws_string_new_from_cursor(auto_ranged_get->base.allocator, &etag_header_value);
+                }
+            } else {
+                AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p ETag headers are missing", (void *)meta_request);
                 aws_raise_error(AWS_ERROR_S3_MISSING_ETAG);
                 error_code = AWS_ERROR_S3_MISSING_ETAG;
                 goto update_synced_data;
             }
+            /* Extract number of parts from ETag and calculate estimated part size */
+            uint32_t num_parts = 0;
+            if (aws_s3_extract_parts_from_etag(etag_header_value, &num_parts) == AWS_OP_SUCCESS && num_parts > 0) {
+                auto_ranged_get->estimated_object_stored_part_size = object_size / num_parts;
 
-            AWS_LOGF_TRACE(
-                AWS_LS_S3_META_REQUEST,
-                "id=%p Etag received for the meta request. value is: " PRInSTR "",
-                (void *)meta_request,
-                AWS_BYTE_CURSOR_PRI(etag_header_value));
-            auto_ranged_get->etag = aws_string_new_from_cursor(auto_ranged_get->base.allocator, &etag_header_value);
+                AWS_LOGF_DEBUG(
+                    AWS_LS_S3_META_REQUEST,
+                    "id=%p Estimated object stored part size: object_size=%" PRIu64 ", num_parts=%" PRIu32
+                    ", estimated_part_size=%" PRIu64,
+                    (void *)meta_request,
+                    object_size,
+                    num_parts,
+                    auto_ranged_get->estimated_object_stored_part_size);
+            } else {
+                /* Failed to parse ETags */
+                aws_raise_error(AWS_ERROR_S3_MISSING_ETAG);
+                error_code = AWS_ERROR_S3_MISSING_ETAG;
+                goto update_synced_data;
+            }
         }
 
         /* If we were able to discover the object-range/content length successfully, then any error code that was passed
          * into this function is being handled and does not indicate an overall failure.*/
         error_code = AWS_ERROR_SUCCESS;
         found_object_size = true;
 
+        if (auto_ranged_get->force_dynamic_part_size ||
+            (!auto_ranged_get->part_size_set && !meta_request->client->part_size_set)) {
+            /* No part size has been set from user. Now we use the optimal part size based on the throughput and memory
+             * limit */
+            uint64_t out_request_optimal_range_size = 0;
+            int error = aws_s3_calculate_request_optimal_range_size(
+                meta_request->client->optimal_range_size,
+                auto_ranged_get->estimated_object_stored_part_size,
+                &out_request_optimal_range_size);
+            if (!error) {
+                /* Override the part size to be optimal */
+                *((size_t *)&meta_request->part_size) = (size_t)out_request_optimal_range_size;
+                if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_HEAD_OBJECT) {
+                    /* Update the first part size as well */
+                    first_part_size = meta_request->part_size;
+                }
+            }
+        }
+
         /* Check for checksums if requested to */
         if (meta_request->checksum_config.validate_response_checksum) {
             if (aws_s3_check_headers_for_checksum(