internal settings change function (#232)

settings change from internal.
- create a queue for pending settings
- when we want to change our settings, we send a setting frame to our peer, then push the pending settings to the back of the queue.
- when the setting ACK is received from the peer, we pop the front one from the queue and apply the settings to our connection.
Co-authored-by: Dengke Tang <[email protected]>

Author: TingDaoK

include/aws/http/private/h2_connection.h

@@ -48,6 +48,10 @@ struct aws_h2_connection {
         /* My settings to send/sent to peer, which affects the decoding */
         uint32_t settings_self[AWS_H2_SETTINGS_END_RANGE];
 
+        /* List using h2_pending_settings.node
+         * Contains settings waiting to be ACKed by peer and applied */
+        struct aws_linked_list pending_settings_queue;
+
         /* Most recent stream-id that was initiated by peer */
         uint32_t latest_peer_initiated_stream_id;
 
@@ -151,6 +155,12 @@ AWS_EXTERN_C_END
 
 /* Private functions called from multiple .c files... */
 
+/* Internal API for changing self settings of the connection */
+int aws_h2_connection_change_settings(
+    struct aws_h2_connection *connection,
+    const struct aws_h2_frame_setting *setting_array,
+    size_t num_settings);
+
 /**
  * Enqueue outgoing frame.
  * Connection takes ownership of frame.

include/aws/http/private/h2_frames.h

@@ -154,8 +154,6 @@ struct aws_h2_frame_encoder {
 
     /* Settings for frame encoder, which is based on the settings received from peer */
     struct {
-        /* the maximum size of the header compression table used to decode header blocks */
-        uint32_t header_table_size;
         /*  the size of the largest frame payload */
         uint32_t max_frame_size;
     } settings;

include/aws/http/private/h2_stream.h

@@ -78,7 +78,7 @@ struct aws_h2_stream *aws_h2_stream_new_request(
 
 enum aws_h2_stream_state aws_h2_stream_get_state(const struct aws_h2_stream *stream);
 
-int aws_h2_stream_window_size_change(struct aws_h2_stream *stream, int32_t size_changed);
+struct aws_h2err aws_h2_stream_window_size_change(struct aws_h2_stream *stream, int32_t size_changed, bool self);
 
 /* Connection is ready to send frames from stream now */
 int aws_h2_stream_on_activated(struct aws_h2_stream *stream, bool *out_has_outgoing_data);

include/aws/http/private/hpack.h

@@ -131,7 +131,10 @@ int aws_hpack_resize_dynamic_table(struct aws_hpack_context *context, size_t new
  * two header blocks. The dynamic table resize and the dynamic table size update entry will be handled properly when we
  * encode the next header block  */
 AWS_HTTP_API
-void aws_hpack_set_max_table_size(struct aws_hpack_context *context, size_t new_max_size);
+void aws_hpack_set_max_table_size(struct aws_hpack_context *context, uint32_t new_max_size);
+
+AWS_HTTP_API
+void aws_hpack_set_protocol_max_size_setting(struct aws_hpack_context *context, uint32_t setting_max_size);
 
 AWS_HTTP_API
 void aws_hpack_set_huffman_mode(struct aws_hpack_context *context, enum aws_hpack_huffman_mode mode);

source/h2_connection.c

@@ -68,6 +68,7 @@ static int s_record_closed_stream(
     struct aws_h2_connection *connection,
     uint32_t stream_id,
     enum aws_h2_stream_closed_when closed_when);
+static void s_try_write_outgoing_frames(struct aws_h2_connection *connection);
 
 static struct aws_h2err s_decoder_on_headers_begin(uint32_t stream_id, void *userdata);
 static struct aws_h2err s_decoder_on_headers_i(
@@ -239,6 +240,7 @@ static struct aws_h2_connection *s_connection_new(
     aws_linked_list_init(&connection->synced_data.pending_stream_list);
 
     aws_linked_list_init(&connection->thread_data.outgoing_streams_list);
+    aws_linked_list_init(&connection->thread_data.pending_settings_queue);
     aws_linked_list_init(&connection->thread_data.stalled_window_streams_list);
     aws_linked_list_init(&connection->thread_data.outgoing_frames_queue);
 
@@ -340,6 +342,12 @@ struct aws_http_connection *aws_http_connection_new_http2_client(
     return &connection->base;
 }
 
+struct h2_pending_settings {
+    struct aws_h2_frame_setting *settings_array;
+    size_t num_settings;
+    struct aws_linked_list_node node;
+};
+
 static void s_handler_destroy(struct aws_channel_handler *handler) {
     struct aws_h2_connection *connection = handler->impl;
     CONNECTION_LOG(TRACE, connection, "Destroying connection");
@@ -348,6 +356,12 @@ static void s_handler_destroy(struct aws_channel_handler *handler) {
     AWS_ASSERT(
         !aws_hash_table_is_valid(&connection->thread_data.active_streams_map) ||
         aws_hash_table_get_entry_count(&connection->thread_data.active_streams_map) == 0);
+    while (!aws_linked_list_empty(&connection->thread_data.pending_settings_queue)) {
+        /* Some settings are sent, but peer never sends ACK back. It's not an error. We just have to clean it up */
+        struct aws_linked_list_node *node = aws_linked_list_pop_front(&connection->thread_data.pending_settings_queue);
+        struct h2_pending_settings *pending_settings = AWS_CONTAINER_OF(node, struct h2_pending_settings, node);
+        aws_mem_release(connection->base.alloc, pending_settings);
+    }
     AWS_ASSERT(aws_linked_list_empty(&connection->thread_data.stalled_window_streams_list));
     AWS_ASSERT(aws_linked_list_empty(&connection->thread_data.outgoing_streams_list));
     AWS_ASSERT(aws_linked_list_empty(&connection->synced_data.pending_stream_list));
@@ -368,6 +382,65 @@ static void s_handler_destroy(struct aws_channel_handler *handler) {
     aws_mem_release(connection->base.alloc, connection);
 }
 
+static struct h2_pending_settings *s_new_pending_settings(
+    struct aws_allocator *allocator,
+    const struct aws_h2_frame_setting *settings_array,
+    size_t num_settings) {
+
+    size_t settings_storage_size = sizeof(struct aws_h2_frame_setting) * num_settings;
+    struct h2_pending_settings *pending_settings;
+    void *settings_storage;
+    if (!aws_mem_acquire_many(
+            allocator,
+            2,
+            &pending_settings,
+            sizeof(struct h2_pending_settings),
+            &settings_storage,
+            settings_storage_size)) {
+        return NULL;
+    }
+
+    AWS_ZERO_STRUCT(*pending_settings);
+    /* We buffer the settings up, incase the caller has freed them when the ACK arrives */
+    pending_settings->settings_array = settings_storage;
+    memcpy(pending_settings->settings_array, settings_array, num_settings * sizeof(*settings_array));
+    pending_settings->num_settings = num_settings;
+
+    return pending_settings;
+}
+
+int aws_h2_connection_change_settings(
+    struct aws_h2_connection *connection,
+    const struct aws_h2_frame_setting *settings_array,
+    size_t num_settings) {
+    AWS_PRECONDITION(aws_channel_thread_is_callers_thread(connection->base.channel_slot->channel));
+    AWS_PRECONDITION(settings_array);
+
+    if (!num_settings) {
+        return AWS_OP_SUCCESS;
+    }
+
+    /* push the setting array into the queue, not applying the change until it is ACKed by peer */
+    struct h2_pending_settings *pending_settings =
+        s_new_pending_settings(connection->base.alloc, settings_array, num_settings);
+    if (!pending_settings) {
+        return AWS_OP_ERR;
+    }
+    /* Send setting frame to inform our peer */
+    struct aws_h2_frame *setting_frame =
+        aws_h2_frame_new_settings(connection->base.alloc, settings_array, num_settings, false /*ACK*/);
+    if (!setting_frame) {
+        CONNECTION_LOGF(ERROR, connection, "Failed to send setting_frames, error %s", aws_error_name(aws_last_error()));
+        aws_mem_release(connection->base.alloc, pending_settings);
+        return AWS_OP_ERR;
+    }
+    aws_h2_connection_enqueue_outgoing_frame(connection, setting_frame);
+
+    aws_linked_list_push_back(&connection->thread_data.pending_settings_queue, &pending_settings->node);
+    s_try_write_outgoing_frames(connection);
+    return AWS_OP_SUCCESS;
+}
+
 void aws_h2_connection_enqueue_outgoing_frame(struct aws_h2_connection *connection, struct aws_h2_frame *frame) {
     AWS_PRECONDITION(frame->type != AWS_H2_FRAME_T_DATA);
     AWS_PRECONDITION(aws_channel_thread_is_callers_thread(connection->base.channel_slot->channel));
@@ -991,6 +1064,7 @@ static struct aws_h2err s_decoder_on_settings(
     size_t num_settings,
     void *userdata) {
     struct aws_h2_connection *connection = userdata;
+    struct aws_h2err err;
     /* Once all values have been processed, the recipient MUST immediately emit a SETTINGS frame with the ACK flag
      * set.(RFC-7540 6.5.3) */
     CONNECTION_LOG(TRACE, connection, "Setting frame processing ends");
@@ -1009,12 +1083,9 @@ static struct aws_h2err s_decoder_on_settings(
             continue;
         }
         switch (settings_array[i].id) {
-            case AWS_H2_SETTINGS_HEADER_TABLE_SIZE:
+            case AWS_H2_SETTINGS_HEADER_TABLE_SIZE: {
                 aws_h2_frame_encoder_set_setting_header_table_size(encoder, settings_array[i].value);
-                break;
-            case AWS_H2_SETTINGS_MAX_FRAME_SIZE:
-                aws_h2_frame_encoder_set_setting_max_frame_size(encoder, settings_array[i].value);
-                break;
+            } break;
             case AWS_H2_SETTINGS_INITIAL_WINDOW_SIZE: {
                 /* When the value of SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST adjust the size of all stream
                  * flow-control windows that it maintains by the difference between the new value and the old value. */
@@ -1024,16 +1095,20 @@ static struct aws_h2err s_decoder_on_settings(
                 while (!aws_hash_iter_done(&stream_iter)) {
                     struct aws_h2_stream *stream = stream_iter.element.value;
                     aws_hash_iter_next(&stream_iter);
-                    if (aws_h2_stream_window_size_change(stream, size_changed)) {
+                    err = aws_h2_stream_window_size_change(stream, size_changed, false /*self*/);
+                    if (aws_h2err_failed(err)) {
                         CONNECTION_LOG(
                             ERROR,
                             connection,
                             "Connection error, change to SETTINGS_INITIAL_WINDOW_SIZE caused a stream's flow-control "
                             "window to exceed the maximum size");
-                        return aws_h2err_from_h2_code(AWS_H2_ERR_FLOW_CONTROL_ERROR);
+                        return err;
                     }
                 }
             } break;
+            case AWS_H2_SETTINGS_MAX_FRAME_SIZE: {
+                aws_h2_frame_encoder_set_setting_max_frame_size(encoder, settings_array[i].value);
+            } break;
         }
         connection->thread_data.settings_peer[settings_array[i].id] = settings_array[i].value;
     }
@@ -1042,14 +1117,58 @@ static struct aws_h2err s_decoder_on_settings(
 
 static struct aws_h2err s_decoder_on_settings_ack(void *userdata) {
     struct aws_h2_connection *connection = userdata;
-    /* #TODO track which SETTINGS frames is ACKed by this */
-
-    /* inform decoder about the settings */
+    if (aws_linked_list_empty(&connection->thread_data.pending_settings_queue)) {
+        CONNECTION_LOG(ERROR, connection, "Connection error, received a malicious extra SETTINGS acknowledgement");
+        return aws_h2err_from_h2_code(AWS_H2_ERR_PROTOCOL_ERROR);
+    }
+    struct aws_h2err err;
+    struct aws_linked_list_node *node = aws_linked_list_front(&connection->thread_data.pending_settings_queue);
+    struct h2_pending_settings *pending_settings = AWS_CONTAINER_OF(node, struct h2_pending_settings, node);
+    struct aws_h2_frame_setting *settings_array = pending_settings->settings_array;
+    /* Apply the settings */
     struct aws_h2_decoder *decoder = connection->thread_data.decoder;
-    uint32_t *settings_self = connection->thread_data.settings_self;
-    aws_h2_decoder_set_setting_header_table_size(decoder, settings_self[AWS_H2_SETTINGS_HEADER_TABLE_SIZE]);
-    aws_h2_decoder_set_setting_enable_push(decoder, settings_self[AWS_H2_SETTINGS_ENABLE_PUSH]);
-    aws_h2_decoder_set_setting_max_frame_size(decoder, settings_self[AWS_H2_SETTINGS_MAX_FRAME_SIZE]);
+    for (size_t i = 0; i < pending_settings->num_settings; i++) {
+        if (connection->thread_data.settings_self[settings_array[i].id] == settings_array[i].value) {
+            /* No change, don't do any work */
+            continue;
+        }
+        switch (settings_array[i].id) {
+            case AWS_H2_SETTINGS_HEADER_TABLE_SIZE: {
+                aws_h2_decoder_set_setting_header_table_size(decoder, settings_array[i].value);
+            } break;
+            case AWS_H2_SETTINGS_ENABLE_PUSH: {
+                aws_h2_decoder_set_setting_enable_push(decoder, settings_array[i].value);
+            } break;
+            case AWS_H2_SETTINGS_INITIAL_WINDOW_SIZE: {
+                /* When the value of SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST adjust the size of all stream
+                 * flow-control windows that it maintains by the difference between the new value and the old value. */
+                int32_t size_changed =
+                    settings_array[i].value - connection->thread_data.settings_self[settings_array[i].id];
+                struct aws_hash_iter stream_iter = aws_hash_iter_begin(&connection->thread_data.active_streams_map);
+                while (!aws_hash_iter_done(&stream_iter)) {
+                    struct aws_h2_stream *stream = stream_iter.element.value;
+                    aws_hash_iter_next(&stream_iter);
+                    err = aws_h2_stream_window_size_change(stream, size_changed, true /*self*/);
+                    if (aws_h2err_failed(err)) {
+                        CONNECTION_LOG(
+                            ERROR,
+                            connection,
+                            "Connection error, change to SETTINGS_INITIAL_WINDOW_SIZE from internal caused a stream's "
+                            "flow-control window to exceed the maximum size");
+                        return err;
+                    }
+                }
+            } break;
+            case AWS_H2_SETTINGS_MAX_FRAME_SIZE: {
+                aws_h2_decoder_set_setting_max_frame_size(decoder, settings_array[i].value);
+            } break;
+        }
+        connection->thread_data.settings_self[settings_array[i].id] = settings_array[i].value;
+    }
+    /* finish applying the settings, remove the front of the queue */
+    aws_linked_list_pop_front(&connection->thread_data.pending_settings_queue);
+    /* clean up the pending_settings */
+    aws_mem_release(connection->base.alloc, pending_settings);
     return AWS_H2ERR_SUCCESS;
 }
 
@@ -1142,20 +1261,6 @@ static int s_send_connection_preface_client_string(struct aws_h2_connection *con
     return AWS_OP_ERR;
 }
 
-/* #TODO actually fill with initial settings */
-/* #TODO track which SETTINGS frames have been ACK'd */
-static int s_enqueue_settings_frame(struct aws_h2_connection *connection) {
-    struct aws_allocator *alloc = connection->base.alloc;
-
-    struct aws_h2_frame *settings_frame = aws_h2_frame_new_settings(alloc, NULL, 0, false /*ack*/);
-    if (!settings_frame) {
-        return AWS_OP_ERR;
-    }
-
-    aws_h2_connection_enqueue_outgoing_frame(connection, settings_frame);
-    return AWS_OP_SUCCESS;
-}
-
 static void s_handler_installed(struct aws_channel_handler *handler, struct aws_channel_slot *slot) {
     AWS_PRECONDITION(aws_channel_thread_is_callers_thread(slot->channel));
     struct aws_h2_connection *connection = handler->impl;
@@ -1169,6 +1274,7 @@ static void s_handler_installed(struct aws_channel_handler *handler, struct aws_
     /* Send HTTP/2 connection preface (RFC-7540 3.5)
      * - clients must send magic string
      * - both client and server must send SETTINGS frame */
+
     if (connection->base.client_data) {
         if (s_send_connection_preface_client_string(connection)) {
             CONNECTION_LOGF(
@@ -1180,16 +1286,22 @@ static void s_handler_installed(struct aws_channel_handler *handler, struct aws_
         }
     }
 
-    if (s_enqueue_settings_frame(connection)) {
+    /* #TODO actually fill with initial settings, now we just disable the push promise for client */
+    /* #TODO Probably we will move the initial settings to connection_new_http2_XXX after we got API for user to change
+     * settings */
+    struct aws_h2_frame_setting initial_settings[2];
+    size_t new_setting_iter = 0;
+    initial_settings[new_setting_iter].id = AWS_H2_SETTINGS_ENABLE_PUSH;
+    initial_settings[new_setting_iter].value = 0;
+    new_setting_iter++;
+    if (aws_h2_connection_change_settings(connection, initial_settings, new_setting_iter)) {
         CONNECTION_LOGF(
             ERROR,
             connection,
             "Failed to send SETTINGS frame for connection preface, %s",
             aws_error_name(aws_last_error()));
         goto error;
     }
-
-    s_try_write_outgoing_frames(connection);
     return;
 
 error:

source/h2_decoder.c

@@ -270,8 +270,6 @@ struct aws_h2_decoder {
 
     /* Settings for decoder, which is based on the settings sent to the peer and ACKed by peer */
     struct {
-        /* the maximum size of the header compression table used to decode header blocks */
-        uint32_t header_table_size;
         /* enable/disable server push */
         uint32_t enable_push;
         /*  the size of the largest frame payload */
@@ -326,7 +324,6 @@ struct aws_h2_decoder *aws_h2_decoder_new(struct aws_h2_decoder_params *params)
         decoder->state = &s_state_prefix;
     }
 
-    decoder->settings.header_table_size = aws_h2_settings_initial[AWS_H2_SETTINGS_HEADER_TABLE_SIZE];
     decoder->settings.enable_push = aws_h2_settings_initial[AWS_H2_SETTINGS_ENABLE_PUSH];
     decoder->settings.max_frame_size = aws_h2_settings_initial[AWS_H2_SETTINGS_MAX_FRAME_SIZE];
 
@@ -1534,7 +1531,8 @@ static struct aws_h2err s_state_fn_connection_preface_string(
 }
 
 void aws_h2_decoder_set_setting_header_table_size(struct aws_h2_decoder *decoder, uint32_t data) {
-    decoder->settings.header_table_size = data;
+    /* Set the protocol_max_size_setting for hpack. */
+    aws_hpack_set_protocol_max_size_setting(decoder->hpack, data);
 }
 
 void aws_h2_decoder_set_setting_enable_push(struct aws_h2_decoder *decoder, uint32_t data) {

source/h2_frames.c

@@ -312,7 +312,6 @@ int aws_h2_frame_encoder_init(
         return AWS_OP_ERR;
     }
 
-    encoder->settings.header_table_size = aws_h2_settings_initial[AWS_H2_SETTINGS_HEADER_TABLE_SIZE];
     encoder->settings.max_frame_size = aws_h2_settings_initial[AWS_H2_SETTINGS_MAX_FRAME_SIZE];
     return AWS_OP_SUCCESS;
 }
@@ -1227,8 +1226,10 @@ int aws_h2_encode_frame(
 }
 
 void aws_h2_frame_encoder_set_setting_header_table_size(struct aws_h2_frame_encoder *encoder, uint32_t data) {
+    /* Setting for dynamic table size changed from peer, we will update the dynamic table size when we encoder the next
+     * header block */
     aws_hpack_set_max_table_size(encoder->hpack, data);
-    encoder->settings.header_table_size = data;
+    aws_hpack_set_protocol_max_size_setting(encoder->hpack, data);
 }
 
 void aws_h2_frame_encoder_set_setting_max_frame_size(struct aws_h2_frame_encoder *encoder, uint32_t data) {