Dynamic default part size

CMakeLists.txt

@@ -108,6 +108,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake"
 
 include(CTest)
 if (BUILD_TESTING)
+    add_definitions(-DAWS_C_S3_ENABLE_TEST_STUBS)
     add_subdirectory(tests)
     if (NOT BYO_CRYPTO AND NOT CMAKE_CROSSCOMPILING)
         add_subdirectory(samples)

include/aws/s3/private/s3_auto_ranged_get.h

@@ -21,6 +21,12 @@ struct aws_s3_auto_ranged_get {
 
     struct aws_string *etag;
 
+    /* Estimated object stored part size based on ETag analysis */
+    uint64_t estimated_object_stored_part_size;
+    /* Part size was set or not from user for this meta request. */
+    bool part_size_set;
+    bool force_dynamic_part_size;
+
     bool initial_message_has_start_range;
     bool initial_message_has_end_range;
     uint64_t initial_range_start;
@@ -74,6 +80,7 @@ AWS_S3_API struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_get_new(
     struct aws_allocator *allocator,
     struct aws_s3_client *client,
     size_t part_size,
+    bool part_size_set,
     const struct aws_s3_meta_request_options *options);
 
 AWS_EXTERN_C_END

include/aws/s3/private/s3_client_impl.h

@@ -178,7 +178,10 @@ struct aws_s3_client_vtable {
         struct aws_http_connection *client_connection,
         const struct aws_http_make_request_options *options);
 
-    void (*after_prepare_upload_part_finish)(struct aws_s3_request *request, struct aws_http_message *message);
+#ifdef AWS_C_S3_ENABLE_TEST_STUBS
+    /********************* TEST ONLY STUB **************************/
+    void (*after_prepare_upload_part_finish_stub)(struct aws_s3_request *request, struct aws_http_message *message);
+#endif
 };
 
 struct aws_s3_upload_part_timeout_stats {
@@ -234,10 +237,17 @@ struct aws_s3_client {
      * to meta requests for use. */
     const size_t part_size;
 
+    bool part_size_set;
+
     /* Size of parts for files when doing gets or puts.  This exists on the client as configurable option that is passed
      * to meta requests for use. */
     const uint64_t max_part_size;
 
+    /* Calculated optimal range size for GET operations based on client configuration (memory limits, throughput
+     * targets). This is used when part_size is not explicitly configured, replacing the default with reasonable
+     * calculation. Value is calculated during client initialization and remains constant for the client's lifetime. */
+    const uint64_t optimal_range_size;
+
     /* File I/O options. */
     bool fio_options_set;
     struct aws_s3_file_io_options fio_opts;

include/aws/s3/private/s3_meta_request_impl.h

@@ -122,6 +122,12 @@ struct aws_s3_meta_request_vtable {
 
     /* Pause the given request */
     int (*pause)(struct aws_s3_meta_request *meta_request, struct aws_s3_meta_request_resume_token **resume_token);
+
+#ifdef AWS_C_S3_ENABLE_TEST_STUBS
+    /********************* TEST ONLY STUB **************************/
+    /* A stub to the update implementation from meta request with the lock held. Only for tests. */
+    bool (*synced_update_stub)(struct aws_s3_meta_request *meta_request);
+#endif
 };
 
 /**
@@ -271,6 +277,9 @@ struct aws_s3_meta_request {
          * This is an optimization, we could have just copied the array when the task runs,
          * but swapping two array-lists back and forth avoids an allocation. */
         struct aws_array_list event_delivery_array;
+
+        /* The range start for the next response body delivery */
+        uint64_t next_deliver_range_start;
     } io_threaded_data;
 
     const bool should_compute_content_md5;
@@ -407,9 +416,11 @@ void aws_s3_meta_request_add_event_for_delivery_synced(
 bool aws_s3_meta_request_are_events_out_for_delivery_synced(struct aws_s3_meta_request *meta_request);
 
 /* Cancel the requests with cancellable HTTP stream for the meta request */
+AWS_S3_API
 void aws_s3_meta_request_cancel_cancellable_requests_synced(struct aws_s3_meta_request *meta_request, int error_code);
 
 /* Cancel the pending buffer futures for the meta request */
+AWS_S3_API
 void aws_s3_meta_request_cancel_pending_buffer_futures_synced(struct aws_s3_meta_request *meta_request, int error_code);
 
 /* Asynchronously read from the meta request's input stream. Should always be done outside of any mutex,

include/aws/s3/private/s3_request.h

@@ -122,8 +122,19 @@ struct aws_s3_request_metrics {
         int error_code;
         /* Retry attempt. */
         uint32_t retry_attempt;
+        /* Is the memory for the request allocated from the buffer pool or not. */
+        bool memory_allocated_from_pool;
     } crt_info_metrics;
 
+    struct {
+        /* Beginning range of this part. */
+        uint64_t part_range_start;
+        /* Last byte of this part. */
+        uint64_t part_range_end;
+        /* Part number that this request refers to. */
+        uint32_t part_number;
+    } part_info_metrics;
+
     struct aws_ref_count ref_count;
 };
 

include/aws/s3/private/s3_util.h

@@ -160,6 +160,15 @@ extern const double g_default_throughput_target_gbps;
 
 AWS_S3_API
 extern const uint64_t g_streaming_object_size_threshold;
+
+AWS_S3_API
+extern const uint64_t g_default_part_size_fallback;
+
+AWS_S3_API
+extern const uint64_t g_default_max_part_size;
+
+AWS_S3_API
+extern const uint64_t g_s3_optimal_range_size_alignment;
 /**
  * Returns AWS_S3_REQUEST_TYPE_UNKNOWN if name doesn't map to an enum value.
  */
@@ -240,12 +249,21 @@ void aws_s3_add_user_agent_header(struct aws_allocator *allocator, struct aws_ht
  * object-size. All output arguments are optional.*/
 AWS_S3_API
 int aws_s3_parse_content_range_response_header(
-    struct aws_allocator *allocator,
     struct aws_http_headers *response_headers,
     uint64_t *out_range_start,
     uint64_t *out_range_end,
     uint64_t *out_object_size);
 
+/* Given a Content-Range header value as a byte cursor, parses the range-start, range-end and
+ * object-size. All output arguments are optional. This is an optimized version that doesn't
+ * require string allocation. */
+AWS_S3_API
+int aws_s3_parse_content_range_cursor(
+    struct aws_byte_cursor content_range_cursor,
+    uint64_t *out_range_start,
+    uint64_t *out_range_end,
+    uint64_t *out_object_size);
+
 /* Given response headers, parses the content-length from a content-length response header.*/
 AWS_S3_API
 int aws_s3_parse_content_length_response_header(
@@ -318,6 +336,51 @@ int aws_s3_check_headers_for_checksum(
     struct aws_byte_buf *out_checksum_buffer,
     bool meta_request_level);
 
+/**
+ * Calculate client-level optimal range size based on memory and connection constraints.
+ * This function is called during client initialization to determine the base range size
+ * using the formula: MemoryLimit / concurrency / divisor.
+ * The result is rounded up to ensure proper alignment and applies minimum size constraints.
+ *
+ * @param memory_limit_in_bytes Total memory limit available for buffering
+ * @param max_connections Maximum number of concurrent connections
+ * @param out_client_optimal_range_size Output parameter for calculated client-level optimal range size
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (caller should fall back to default)
+ */
+AWS_S3_API
+int aws_s3_calculate_client_optimal_range_size(
+    uint64_t memory_limit_in_bytes,
+    uint32_t max_connections,
+    uint64_t *out_client_optimal_range_size);
+
+/**
+ * Calculate request-level optimal range size by considering object-specific information.
+ * This function is called per request to adjust the client-level range size based on
+ * estimated object stored part size using: min(client_optimal_range_size, estimated_object_stored_part_size).
+ *
+ * @param client_optimal_range_size The client-level optimal range size from initialization
+ * @param estimated_object_stored_part_size Estimated size of object stored parts in S3
+ * @param out_request_optimal_range_size Output parameter for calculated request-level optimal range size
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (caller should fall back to client size)
+ */
+AWS_S3_API
+int aws_s3_calculate_request_optimal_range_size(
+    uint64_t client_optimal_range_size,
+    uint64_t estimated_object_stored_part_size,
+    uint64_t *out_request_optimal_range_size);
+
+/**
+ * Extract the number of parts from an S3 ETag header value.
+ * S3 multipart upload ETags have the format "<hash>-<number_of_parts>".
+ * Single-part uploads have ETags without dashes.
+ *
+ * @param etag_header_value The ETag header value (may include quotes)
+ * @param out_num_parts Output parameter for the number of parts (1 for single-part uploads)
+ * @return AWS_OP_SUCCESS on success, AWS_OP_ERR on failure (invalid ETag format)
+ */
+AWS_S3_API
+int aws_s3_extract_parts_from_etag(struct aws_byte_cursor etag_header_value, uint32_t *out_num_parts);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_UTIL_H */

include/aws/s3/s3_client.h

@@ -510,7 +510,9 @@ struct aws_s3_client_config {
      * Optional.
      * Size of parts the object will be downloaded or uploaded in, in bytes.
      * This only affects AWS_S3_META_REQUEST_TYPE_GET_OBJECT and AWS_S3_META_REQUEST_TYPE_PUT_OBJECT.
-     * If not set, this defaults to 8 MiB.
+     *
+     * If not set, a dynamic default part size will be used based on the throughput target, memory_limit_in_bytes.
+     *
      * The client will adjust the part size for AWS_S3_META_REQUEST_TYPE_PUT_OBJECT if needed for service limits (max
      * number of parts per upload is 10,000, minimum upload part size is 5 MiB).
      *
@@ -875,13 +877,22 @@ struct aws_s3_meta_request_options {
      * Optional.
      * Size of parts the object will be downloaded or uploaded in, in bytes.
      * This only affects AWS_S3_META_REQUEST_TYPE_GET_OBJECT and AWS_S3_META_REQUEST_TYPE_PUT_OBJECT.
-     * If not set, the value from `aws_s3_client_config.part_size` is used, which defaults to 8MiB.
+     *
+     * If not set, the value from `aws_s3_client_config.part_size` is used, which defaults to a dynamic value based on
+     * the throughput target, memory_limit_in_bytes and the requested object size.
      *
      * The client will adjust the part size for AWS_S3_META_REQUEST_TYPE_PUT_OBJECT if needed for service limits (max
      * number of parts per upload is 10,000, minimum upload part size is 5 MiB).
      */
     uint64_t part_size;
 
+    /**
+     * Optional - EXPERIMENTAL/UNSTABLE
+     * Set this to prefer for the dynamic default part_size over the part size set for both client and meta request for
+     * the best performance under the memory constrain, especially for getting large objects.
+     */
+    bool force_dynamic_part_size;
+
     /**
      * Optional.
      * The size threshold in bytes for when to use multipart uploads.
@@ -1604,6 +1615,26 @@ int aws_s3_request_metrics_get_error_code(const struct aws_s3_request_metrics *m
 AWS_S3_API
 uint32_t aws_s3_request_metrics_get_retry_attempt(const struct aws_s3_request_metrics *metrics);
 
+/* Get the memory for the request allocated from the pool or not. Cannot fail */
+AWS_S3_API
+bool aws_s3_request_metrics_get_memory_allocated_from_pool(const struct aws_s3_request_metrics *metrics);
+
+/* Get the beginning range of this part from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_range_start(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_part_range_start);
+
+/* Get the last byte of this part from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_range_end(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_part_range_end);
+
+/* Get the part number from request metrics. */
+AWS_S3_API
+void aws_s3_request_metrics_get_part_number(const struct aws_s3_request_metrics *metrics, uint32_t *out_part_number);
+
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL