From b265ed4690aad16ed31d0bfc89ebd112c7f9575b Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 24 Apr 2025 14:36:05 -0700 Subject: [PATCH 01/14] update aws_event_loop_get_base_event_loop_group to get event loop group from the event loop --- include/aws/io/event_loop.h | 1 - include/aws/io/private/event_loop_impl.h | 8 ++++---- source/bsd/kqueue_event_loop.c | 13 +++---------- source/darwin/dispatch_queue_event_loop.c | 15 +-------------- source/darwin/dispatch_queue_event_loop_private.h | 1 - source/darwin/nw_socket.c | 4 ++-- source/event_loop.c | 5 ++--- source/linux/epoll_event_loop.c | 11 +---------- source/windows/iocp/iocp_event_loop.c | 12 ++---------- 9 files changed, 15 insertions(+), 55 deletions(-) diff --git a/include/aws/io/event_loop.h b/include/aws/io/event_loop.h index 61421bf4b..e27c91d1f 100644 --- a/include/aws/io/event_loop.h +++ b/include/aws/io/event_loop.h @@ -46,7 +46,6 @@ struct aws_event_loop_vtable { void *user_data); int (*unsubscribe_from_io_events)(struct aws_event_loop *event_loop, struct aws_io_handle *handle); void (*free_io_event_resources)(void *user_data); - void *(*get_base_event_loop_group)(struct aws_event_loop *event_loop); bool (*is_on_callers_thread)(struct aws_event_loop *event_loop); }; diff --git a/include/aws/io/private/event_loop_impl.h b/include/aws/io/private/event_loop_impl.h index bba9653c5..d09f07741 100644 --- a/include/aws/io/private/event_loop_impl.h +++ b/include/aws/io/private/event_loop_impl.h @@ -75,6 +75,7 @@ struct aws_event_loop { uint64_t latest_tick_start; size_t current_tick_latency_sum; struct aws_atomic_var next_flush_time; + struct aws_event_loop_group *base_elg; void *impl_data; }; @@ -322,12 +323,11 @@ AWS_IO_API void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, struct aws_io_handle *handle); /** - * Retrieves the aws_event_loop_group that is the parent of the aws_event_loop. This is only supported when using a - * dispatch queue event loop as they are async and their sockets need to retain a refcount on the elg to keep it alive - * and insure it has not been asyncronously destroyed before anything that needs it. + * Retrieves the aws_event_loop_group that is the parent of the aws_event_loop. return null if the event loop group + * is not available. */ AWS_IO_API -void *get_base_event_loop_group(struct aws_event_loop *event_loop); +void *aws_event_loop_get_base_event_loop_group(struct aws_event_loop *event_loop); AWS_IO_API struct aws_event_loop_group *aws_event_loop_group_new_internal( diff --git a/source/bsd/kqueue_event_loop.c b/source/bsd/kqueue_event_loop.c index 6fca33059..d62ed1320 100644 --- a/source/bsd/kqueue_event_loop.c +++ b/source/bsd/kqueue_event_loop.c @@ -49,15 +49,7 @@ static int s_subscribe_to_io_events( void *user_data); static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struct aws_io_handle *handle); static void s_free_io_event_resources(void *user_data); -static void *s_get_base_event_loop_group(struct aws_event_loop *event_loop) { - (void)event_loop; - AWS_LOGF_ERROR( - AWS_LS_IO_EVENT_LOOP, - "id=%p: get_base_event_loop_group() is not supported using KQueue Event Loops", - (void *)event_loop); - aws_raise_error(AWS_ERROR_PLATFORM_NOT_SUPPORTED); - return NULL; -} + static bool s_is_event_thread(struct aws_event_loop *event_loop); static void aws_event_loop_thread(void *user_data); @@ -148,7 +140,6 @@ struct aws_event_loop_vtable s_kqueue_vtable = { .subscribe_to_io_events = s_subscribe_to_io_events, .unsubscribe_from_io_events = s_unsubscribe_from_io_events, .free_io_event_resources = s_free_io_event_resources, - .get_base_event_loop_group = s_get_base_event_loop_group, .is_on_callers_thread = s_is_event_thread, }; @@ -275,6 +266,8 @@ struct aws_event_loop *aws_event_loop_new_with_kqueue( event_loop->vtable = &s_kqueue_vtable; + event_loop->base_elg = options->parent_elg; + /* success */ return event_loop; diff --git a/source/darwin/dispatch_queue_event_loop.c b/source/darwin/dispatch_queue_event_loop.c index bfb110c1b..5dda49783 100644 --- a/source/darwin/dispatch_queue_event_loop.c +++ b/source/darwin/dispatch_queue_event_loop.c @@ -64,7 +64,6 @@ static void s_free_io_event_resources(void *user_data) { /* No io event resources to free */ (void)user_data; } -static void *s_get_base_event_loop_group(struct aws_event_loop *event_loop); static bool s_is_on_callers_thread(struct aws_event_loop *event_loop); static struct aws_event_loop_vtable s_vtable = { @@ -80,7 +79,6 @@ static struct aws_event_loop_vtable s_vtable = { .subscribe_to_io_events = s_subscribe_to_io_events, .unsubscribe_from_io_events = s_unsubscribe_from_io_events, .free_io_event_resources = s_free_io_event_resources, - .get_base_event_loop_group = s_get_base_event_loop_group, .is_on_callers_thread = s_is_on_callers_thread, }; @@ -254,7 +252,7 @@ struct aws_event_loop *aws_event_loop_new_with_dispatch_queue( dispatch_loop->allocator = alloc; loop->impl_data = dispatch_loop; dispatch_loop->base_loop = loop; - dispatch_loop->base_elg = options->parent_elg; + dispatch_loop->base_loop->base_elg = options->parent_elg; dispatch_loop->synced_data.execution_state = AWS_DLES_SUSPENDED; aws_ref_count_init(&dispatch_loop->ref_count, dispatch_loop, s_dispatch_event_loop_on_zero_ref_count); @@ -738,17 +736,6 @@ static int s_connect_to_io_completion_port(struct aws_event_loop *event_loop, st return AWS_OP_SUCCESS; } -/* - * Because dispatch queue is async we may need to acquire a refcount of the parent event loop group to prevent - * the event loop or dispatch loop from being cleaned out from underneath something that needs it. We expose the - * base elg so anything that needs to insure the event loops and dispatch loops don't get prematurely cleaned can - * hold a refcount. - */ -static void *s_get_base_event_loop_group(struct aws_event_loop *event_loop) { - struct aws_dispatch_loop *dispatch_loop = event_loop->impl_data; - return dispatch_loop->base_elg; -} - /* * We use aws_thread_id_equal with syched_data.current_thread_id and synced_data.is_executing to determine * if operation is being executed on the same dispatch queue thread. diff --git a/source/darwin/dispatch_queue_event_loop_private.h b/source/darwin/dispatch_queue_event_loop_private.h index c845f2cfe..2ead6bb42 100644 --- a/source/darwin/dispatch_queue_event_loop_private.h +++ b/source/darwin/dispatch_queue_event_loop_private.h @@ -35,7 +35,6 @@ struct aws_dispatch_loop { dispatch_queue_t dispatch_queue; struct aws_task_scheduler scheduler; struct aws_event_loop *base_loop; - struct aws_event_loop_group *base_elg; struct aws_ref_count ref_count; diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index 7489a8001..a1e230300 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -424,7 +424,7 @@ static void s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loo "id=%p nw_socket=%p: s_set_event_loop: socket acquire event loop group.", (void *)aws_socket, (void *)nw_socket); - aws_event_loop_group_acquire(get_base_event_loop_group(event_loop)); + aws_event_loop_group_acquire(aws_event_loop_get_base_event_loop_group(event_loop)); } static void s_release_event_loop(struct nw_socket *nw_socket) { @@ -433,7 +433,7 @@ static void s_release_event_loop(struct nw_socket *nw_socket) { AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket has not event loop.", (void *)nw_socket); return; } - aws_event_loop_group_release(get_base_event_loop_group(nw_socket->event_loop)); + aws_event_loop_group_release(aws_event_loop_get_base_event_loop_group(nw_socket->event_loop)); AWS_LOGF_DEBUG( AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket release event loop group.", (void *)nw_socket); nw_socket->event_loop = NULL; diff --git a/source/event_loop.c b/source/event_loop.c index d7911bd95..048ea6cca 100644 --- a/source/event_loop.c +++ b/source/event_loop.c @@ -655,9 +655,8 @@ void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, s event_loop->vtable->free_io_event_resources(handle->additional_data); } -void *get_base_event_loop_group(struct aws_event_loop *event_loop) { - AWS_ASSERT(event_loop && event_loop->vtable->get_base_event_loop_group); - return event_loop->vtable->get_base_event_loop_group(event_loop); +void aws_event_loop_get_base_event_loop_group(struct aws_event_loop *event_loop) { + return event_loop->base_elg; } bool aws_event_loop_thread_is_callers_thread(struct aws_event_loop *event_loop) { diff --git a/source/linux/epoll_event_loop.c b/source/linux/epoll_event_loop.c index ea440ee89..676623ace 100644 --- a/source/linux/epoll_event_loop.c +++ b/source/linux/epoll_event_loop.c @@ -68,15 +68,6 @@ static int s_subscribe_to_io_events( void *user_data); static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struct aws_io_handle *handle); static void s_free_io_event_resources(void *user_data); -static void *s_get_base_event_loop_group(struct aws_event_loop *event_loop) { - (void)event_loop; - AWS_LOGF_ERROR( - AWS_LS_IO_EVENT_LOOP, - "id=%p: get_base_event_loop_group() is not supported using Epoll Event Loops", - (void *)event_loop); - aws_raise_error(AWS_ERROR_PLATFORM_NOT_SUPPORTED); - return NULL; -} static bool s_is_on_callers_thread(struct aws_event_loop *event_loop); static void aws_event_loop_thread(void *args); @@ -94,7 +85,6 @@ static struct aws_event_loop_vtable s_vtable = { .subscribe_to_io_events = s_subscribe_to_io_events, .unsubscribe_from_io_events = s_unsubscribe_from_io_events, .free_io_event_resources = s_free_io_event_resources, - .get_base_event_loop_group = s_get_base_event_loop_group, .is_on_callers_thread = s_is_on_callers_thread, }; @@ -218,6 +208,7 @@ struct aws_event_loop *aws_event_loop_new_with_epoll( loop->impl_data = epoll_loop; loop->vtable = &s_vtable; + loop->base_elg = options->parent_elg; return loop; diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c index bd31cfa77..24a4b68d1 100644 --- a/source/windows/iocp/iocp_event_loop.c +++ b/source/windows/iocp/iocp_event_loop.c @@ -124,15 +124,6 @@ static int s_subscribe_to_io_events( } static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struct aws_io_handle *handle); static void s_free_io_event_resources(void *user_data); -static void *s_get_base_event_loop_group(struct aws_event_loop *event_loop) { - (void)event_loop; - AWS_LOGF_ERROR( - AWS_LS_IO_EVENT_LOOP, - "id=%p: get_base_event_loop_group() is not supported using IOCP Event Loops", - (void *)event_loop); - aws_raise_error(AWS_ERROR_PLATFORM_NOT_SUPPORTED); - return NULL; -} static void aws_event_loop_thread(void *user_data); void aws_overlapped_init( @@ -169,7 +160,6 @@ struct aws_event_loop_vtable s_iocp_vtable = { .subscribe_to_io_events = s_subscribe_to_io_events, .unsubscribe_from_io_events = s_unsubscribe_from_io_events, .free_io_event_resources = s_free_io_event_resources, - .get_base_event_loop_group = s_get_base_event_loop_group, .is_on_callers_thread = s_is_event_thread, }; @@ -273,6 +263,8 @@ struct aws_event_loop *aws_event_loop_new_with_iocp( event_loop->vtable = &s_iocp_vtable; + event_loop->base_elg = options->parent_elg; + return event_loop; clean_up: From cd37e8ca8d1bd72914ed6d9354750d3ca893948c Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 24 Apr 2025 14:55:30 -0700 Subject: [PATCH 02/14] acquire event loop --- include/aws/io/private/event_loop_impl.h | 13 ++++++++++--- source/darwin/nw_socket.c | 17 +++++++++++------ source/event_loop.c | 10 ++++++++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/include/aws/io/private/event_loop_impl.h b/include/aws/io/private/event_loop_impl.h index d09f07741..5cad0f5df 100644 --- a/include/aws/io/private/event_loop_impl.h +++ b/include/aws/io/private/event_loop_impl.h @@ -323,11 +323,18 @@ AWS_IO_API void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, struct aws_io_handle *handle); /** - * Retrieves the aws_event_loop_group that is the parent of the aws_event_loop. return null if the event loop group - * is not available. + * "acquire a reference to the event loop" by acquiring a reference to the base event loop group it belongs to. + * This is used to ensure that the event loop alive. Return null if the event loop does not belong to a group, + * otherwise, return the event loop. */ AWS_IO_API -void *aws_event_loop_get_base_event_loop_group(struct aws_event_loop *event_loop); +struct aws_event_loop *aws_event_loop_acquire(struct aws_event_loop *event_loop); + +/** + * "release a reference to the event loop" by releasing a reference to the base event loop group it belongs to. + */ +AWS_IO_API +void aws_event_loop_release(struct aws_event_loop *event_loop); AWS_IO_API struct aws_event_loop_group *aws_event_loop_group_new_internal( diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index 6ce396867..6bbaa2f40 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -420,11 +420,16 @@ static void s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loo nw_socket->event_loop = event_loop; AWS_LOGF_DEBUG( - AWS_LS_IO_SOCKET, - "id=%p nw_socket=%p: s_set_event_loop: socket acquire event loop group.", - (void *)aws_socket, - (void *)nw_socket); - aws_event_loop_group_acquire(aws_event_loop_get_base_event_loop_group(event_loop)); + AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: nw_socket set event loop.", (void *)aws_socket, (void *)nw_socket); + + if (!aws_event_loop_acquire(event_loop)) { + AWS_LOGF_ERROR( + AWS_LS_IO_SOCKET, + "id=%p nw_socket=%p: failed to acquire event loop group.", + (void *)aws_socket, + (void *)nw_socket); + AWS_FATAL_ASSERT(0); + } } static void s_release_event_loop(struct nw_socket *nw_socket) { @@ -433,7 +438,7 @@ static void s_release_event_loop(struct nw_socket *nw_socket) { AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket has not event loop.", (void *)nw_socket); return; } - aws_event_loop_group_release(aws_event_loop_get_base_event_loop_group(nw_socket->event_loop)); + aws_event_loop_release(nw_socket->event_loop); AWS_LOGF_DEBUG( AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket release event loop group.", (void *)nw_socket); nw_socket->event_loop = NULL; diff --git a/source/event_loop.c b/source/event_loop.c index 048ea6cca..c29ba3f18 100644 --- a/source/event_loop.c +++ b/source/event_loop.c @@ -655,8 +655,14 @@ void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, s event_loop->vtable->free_io_event_resources(handle->additional_data); } -void aws_event_loop_get_base_event_loop_group(struct aws_event_loop *event_loop) { - return event_loop->base_elg; +void aws_event_loop_acquire(struct aws_event_loop *event_loop) { + AWS_ASSERT(event_loop->base_elg); + return aws_event_loop_group_acquire(event_loop->base_elg); +} + +void aws_event_loop_release(struct aws_event_loop *event_loop) { + AWS_ASSERT(event_loop->base_elg); + return aws_event_loop_group_release(event_loop->base_elg); } bool aws_event_loop_thread_is_callers_thread(struct aws_event_loop *event_loop) { From 0f49e4a1c2417acf74ec5220b5803f09686c0964 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 24 Apr 2025 15:09:24 -0700 Subject: [PATCH 03/14] fix --- source/event_loop.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/event_loop.c b/source/event_loop.c index c29ba3f18..be1379e23 100644 --- a/source/event_loop.c +++ b/source/event_loop.c @@ -655,14 +655,15 @@ void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, s event_loop->vtable->free_io_event_resources(handle->additional_data); } -void aws_event_loop_acquire(struct aws_event_loop *event_loop) { +struct aws_event_loop *aws_event_loop_acquire(struct aws_event_loop *event_loop) { AWS_ASSERT(event_loop->base_elg); - return aws_event_loop_group_acquire(event_loop->base_elg); + aws_event_loop_group_acquire(event_loop->base_elg); + return event_loop; } void aws_event_loop_release(struct aws_event_loop *event_loop) { AWS_ASSERT(event_loop->base_elg); - return aws_event_loop_group_release(event_loop->base_elg); + aws_event_loop_group_release(event_loop->base_elg); } bool aws_event_loop_thread_is_callers_thread(struct aws_event_loop *event_loop) { From 4a96a41b3f0650bcde94ef7fab4082dc1979aafe Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 24 Apr 2025 15:31:15 -0700 Subject: [PATCH 04/14] update source/linux/epoll_event_loop.c checksum --- .github/workflows/proof-alarm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/proof-alarm.yml b/.github/workflows/proof-alarm.yml index 0f06c333b..e44160ed4 100644 --- a/.github/workflows/proof-alarm.yml +++ b/.github/workflows/proof-alarm.yml @@ -16,7 +16,7 @@ jobs: - name: Check run: | TMPFILE=$(mktemp) - echo "e857a2e5f72ab77a94e56372d89abf99 source/linux/epoll_event_loop.c" > $TMPFILE + echo "844fcb2768afd3ab013fa49447c1b855 source/linux/epoll_event_loop.c" > $TMPFILE md5sum --check $TMPFILE # No further steps if successful From 4d853b333332c1a3510569b526b28f99323096b4 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 24 Apr 2025 15:45:36 -0700 Subject: [PATCH 05/14] update vcc epoll event loop --- tests/vcc/new_destroy.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/vcc/new_destroy.c b/tests/vcc/new_destroy.c index fdb5a788d..30de44497 100644 --- a/tests/vcc/new_destroy.c +++ b/tests/vcc/new_destroy.c @@ -184,6 +184,7 @@ struct aws_event_loop *aws_event_loop_new_with_epoll( loop->impl_data = epoll_loop; loop->vtable = &s_vtable; + loop->base_elg = options->parent_elg; _(wrap(&epoll_loop->task_pre_queue.head)) _(wrap(&epoll_loop->task_pre_queue.tail)) From 145d71c204da18922276d7625f2c84d194f8a644 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Fri, 25 Apr 2025 09:19:14 -0700 Subject: [PATCH 06/14] rename the function and move it to public header --- include/aws/io/event_loop.h | 16 ++++++++++++++++ include/aws/io/private/event_loop_impl.h | 14 -------------- source/darwin/nw_socket.c | 4 ++-- source/event_loop.c | 21 ++++++++++----------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/include/aws/io/event_loop.h b/include/aws/io/event_loop.h index e27c91d1f..7bcea49f5 100644 --- a/include/aws/io/event_loop.h +++ b/include/aws/io/event_loop.h @@ -174,6 +174,22 @@ struct aws_event_loop_group *aws_event_loop_group_acquire(struct aws_event_loop_ AWS_IO_API void aws_event_loop_group_release(struct aws_event_loop_group *el_group); +/** + * Increments the reference count on the event loop group from event loop, allowing the caller to take a reference to + * it. + * + * Returns the base event loop group of the event loop, or null if the event loop does not belong to a group. + */ +AWS_IO_API +struct aws_event_loop_group *aws_event_loop_group_acquire_from_event_loop(struct aws_event_loop *event_loop); + +/** + * Decrements the ref count of the event loop's base event loop group. When the ref count drops to zero, the event loop + * group will be destroyed. + */ +AWS_IO_API +void aws_event_loop_group_release_from_event_loop(struct aws_event_loop *event_loop); + /** * Returns the event loop at a particular index. If the index is out of bounds, null is returned. */ diff --git a/include/aws/io/private/event_loop_impl.h b/include/aws/io/private/event_loop_impl.h index 5cad0f5df..3c333305d 100644 --- a/include/aws/io/private/event_loop_impl.h +++ b/include/aws/io/private/event_loop_impl.h @@ -322,20 +322,6 @@ int aws_event_loop_unsubscribe_from_io_events(struct aws_event_loop *event_loop, AWS_IO_API void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, struct aws_io_handle *handle); -/** - * "acquire a reference to the event loop" by acquiring a reference to the base event loop group it belongs to. - * This is used to ensure that the event loop alive. Return null if the event loop does not belong to a group, - * otherwise, return the event loop. - */ -AWS_IO_API -struct aws_event_loop *aws_event_loop_acquire(struct aws_event_loop *event_loop); - -/** - * "release a reference to the event loop" by releasing a reference to the base event loop group it belongs to. - */ -AWS_IO_API -void aws_event_loop_release(struct aws_event_loop *event_loop); - AWS_IO_API struct aws_event_loop_group *aws_event_loop_group_new_internal( struct aws_allocator *allocator, diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index 6bbaa2f40..cf24773cd 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -422,7 +422,7 @@ static void s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loo AWS_LOGF_DEBUG( AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: nw_socket set event loop.", (void *)aws_socket, (void *)nw_socket); - if (!aws_event_loop_acquire(event_loop)) { + if (!aws_event_loop_group_acquire_from_event_loop(event_loop)) { AWS_LOGF_ERROR( AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: failed to acquire event loop group.", @@ -438,7 +438,7 @@ static void s_release_event_loop(struct nw_socket *nw_socket) { AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket has not event loop.", (void *)nw_socket); return; } - aws_event_loop_release(nw_socket->event_loop); + aws_event_loop_group_release_from_event_loop(nw_socket->event_loop); AWS_LOGF_DEBUG( AWS_LS_IO_SOCKET, "nw_socket=%p: s_release_event_loop: socket release event loop group.", (void *)nw_socket); nw_socket->event_loop = NULL; diff --git a/source/event_loop.c b/source/event_loop.c index be1379e23..88ce5e21b 100644 --- a/source/event_loop.c +++ b/source/event_loop.c @@ -380,6 +380,16 @@ void aws_event_loop_group_release(struct aws_event_loop_group *el_group) { } } +struct aws_event_loop_group *aws_event_loop_group_acquire_from_event_loop(struct aws_event_loop *event_loop) { + AWS_ASSERT(event_loop); + return aws_event_loop_group_acquire(event_loop->base_elg); +} + +void aws_event_loop_group_release_from_event_loop(struct aws_event_loop *event_loop) { + AWS_ASSERT(event_loop); + aws_event_loop_group_release(event_loop->base_elg); +} + size_t aws_event_loop_group_get_loop_count(const struct aws_event_loop_group *el_group) { return aws_array_list_length(&el_group->event_loops); } @@ -655,17 +665,6 @@ void aws_event_loop_free_io_event_resources(struct aws_event_loop *event_loop, s event_loop->vtable->free_io_event_resources(handle->additional_data); } -struct aws_event_loop *aws_event_loop_acquire(struct aws_event_loop *event_loop) { - AWS_ASSERT(event_loop->base_elg); - aws_event_loop_group_acquire(event_loop->base_elg); - return event_loop; -} - -void aws_event_loop_release(struct aws_event_loop *event_loop) { - AWS_ASSERT(event_loop->base_elg); - aws_event_loop_group_release(event_loop->base_elg); -} - bool aws_event_loop_thread_is_callers_thread(struct aws_event_loop *event_loop) { AWS_ASSERT(event_loop->vtable && event_loop->vtable->is_on_callers_thread); return event_loop->vtable->is_on_callers_thread(event_loop); From c8bde37c389b4eac99283495087ad89a55251403 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Fri, 25 Apr 2025 09:36:46 -0700 Subject: [PATCH 07/14] handle set event loop failure --- source/darwin/dispatch_queue_event_loop.c | 2 +- source/darwin/nw_socket.c | 51 +++++++++++++++++------ source/windows/iocp/iocp_event_loop.c | 1 - 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/source/darwin/dispatch_queue_event_loop.c b/source/darwin/dispatch_queue_event_loop.c index 5dda49783..f66a7804f 100644 --- a/source/darwin/dispatch_queue_event_loop.c +++ b/source/darwin/dispatch_queue_event_loop.c @@ -247,12 +247,12 @@ struct aws_event_loop *aws_event_loop_new_with_dispatch_queue( } loop->vtable = &s_vtable; + loop->base_elg = options->parent_elg; dispatch_loop = aws_mem_calloc(alloc, 1, sizeof(struct aws_dispatch_loop)); dispatch_loop->allocator = alloc; loop->impl_data = dispatch_loop; dispatch_loop->base_loop = loop; - dispatch_loop->base_loop->base_elg = options->parent_elg; dispatch_loop->synced_data.execution_state = AWS_DLES_SUSPENDED; aws_ref_count_init(&dispatch_loop->ref_count, dispatch_loop, s_dispatch_event_loop_on_zero_ref_count); diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index cf24773cd..1eff1a4a7 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -412,24 +412,28 @@ static bool s_validate_event_loop(struct aws_event_loop *event_loop) { return event_loop && event_loop->vtable && event_loop->impl_data; } -static void s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loop *event_loop) { +static bool s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loop *event_loop) { aws_socket->event_loop = event_loop; struct nw_socket *nw_socket = aws_socket->impl; // Never re-assign an event loop AWS_FATAL_ASSERT(nw_socket->event_loop == NULL); - nw_socket->event_loop = event_loop; - - AWS_LOGF_DEBUG( - AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: nw_socket set event loop.", (void *)aws_socket, (void *)nw_socket); + // Acquire the event loop group from the event loop. The event loop group will be released when the socket is + // destroyed. if (!aws_event_loop_group_acquire_from_event_loop(event_loop)) { AWS_LOGF_ERROR( AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: failed to acquire event loop group.", (void *)aws_socket, (void *)nw_socket); - AWS_FATAL_ASSERT(0); + return false; } + + nw_socket->event_loop = event_loop; + AWS_LOGF_DEBUG( + AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: nw_socket set event loop.", (void *)aws_socket, (void *)nw_socket); + + return true; } static void s_release_event_loop(struct nw_socket *nw_socket) { @@ -1803,7 +1807,10 @@ static int s_socket_connect_fn(struct aws_socket *socket, struct aws_socket_conn } /* event_loop must be set prior to setup of socket parameters. */ - s_set_event_loop(socket, event_loop); + if (!s_set_event_loop(nw_socket, event_loop)) { + goto error; + } + if (s_setup_socket_params(nw_socket, &socket->options)) { goto error; } @@ -2321,6 +2328,18 @@ static int s_socket_start_accept_fn( return aws_raise_error(AWS_IO_EVENT_LOOP_ALREADY_ASSIGNED); } + if (!s_set_event_loop(socket, accept_loop)) { + AWS_LOGF_ERROR( + AWS_LS_IO_SOCKET, + "id=%p handle=%p: failed to set event loop %p, invalid event loop. It is most likely the event loop does " + "not has a parent event loop group.", + (void *)socket, + socket->io_handle.data.handle, + (void *)accept_loop); + s_unlock_socket_synced_data(nw_socket); + return aws_raise_error(AWS_IO_SOCKET_INVALID_OPTIONS); + } + aws_event_loop_connect_handle_to_io_completion_port(accept_loop, &socket->io_handle); socket->accept_result_fn = options.on_accept_result; socket->connect_accept_user_data = options.on_accept_result_user_data; @@ -2328,8 +2347,6 @@ static int s_socket_start_accept_fn( nw_socket->on_accept_started_fn = options.on_accept_start; nw_socket->listen_accept_started_user_data = options.on_accept_start_user_data; - s_set_event_loop(socket, accept_loop); - nw_listener_set_state_changed_handler( socket->io_handle.data.handle, ^(nw_listener_state_t state, nw_error_t error) { s_handle_listener_state_changed_fn(nw_socket, state, error); @@ -2438,9 +2455,20 @@ static int s_socket_assign_to_event_loop_fn(struct aws_socket *socket, struct aw socket->io_handle.data.handle, (void *)event_loop); - if (aws_event_loop_connect_handle_to_io_completion_port(event_loop, &socket->io_handle)) { + if (!s_set_event_loop(socket, event_loop)) { + AWS_LOGF_ERROR( + AWS_LS_IO_SOCKET, + "id=%p handle=%p: assigning event loop %p failed. Invalid event loop. It is likely the event loop " + "does not has a parent event loop group.", + (void *)socket, + socket->io_handle.data.handle, + (void *)event_loop); + aws_raise_error(AWS_IO_SOCKET_INVALID_OPTIONS); + return AWS_IO_SOCKET_INVALID_OPTIONS; + } - AWS_LOGF_DEBUG( + if (aws_event_loop_connect_handle_to_io_completion_port(event_loop, &socket->io_handle)) { + AWS_LOGF_ERROR( AWS_LS_IO_SOCKET, "id=%p handle=%p: assigning event loop %p failed", (void *)socket, @@ -2449,7 +2477,6 @@ static int s_socket_assign_to_event_loop_fn(struct aws_socket *socket, struct aw return AWS_OP_ERR; } - s_set_event_loop(socket, event_loop); nw_connection_start(socket->io_handle.data.handle); return AWS_OP_SUCCESS; } diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c index 24a4b68d1..3e932ecfb 100644 --- a/source/windows/iocp/iocp_event_loop.c +++ b/source/windows/iocp/iocp_event_loop.c @@ -262,7 +262,6 @@ struct aws_event_loop *aws_event_loop_new_with_iocp( event_loop->impl_data = impl; event_loop->vtable = &s_iocp_vtable; - event_loop->base_elg = options->parent_elg; return event_loop; From e65cf0977c093bb93d246f669f84793b4fdead92 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Fri, 25 Apr 2025 09:50:30 -0700 Subject: [PATCH 08/14] fix socket --- source/darwin/nw_socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index 1eff1a4a7..5c94fd9ba 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -1807,7 +1807,7 @@ static int s_socket_connect_fn(struct aws_socket *socket, struct aws_socket_conn } /* event_loop must be set prior to setup of socket parameters. */ - if (!s_set_event_loop(nw_socket, event_loop)) { + if (!s_set_event_loop(socket, event_loop)) { goto error; } From 248d7b65e0c26b9b76005ebfad9571a5c9ab5214 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 28 Apr 2025 09:18:23 -0700 Subject: [PATCH 09/14] fix style, use int for return value --- source/darwin/nw_socket.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/darwin/nw_socket.c b/source/darwin/nw_socket.c index 5c94fd9ba..1517cee48 100644 --- a/source/darwin/nw_socket.c +++ b/source/darwin/nw_socket.c @@ -412,7 +412,7 @@ static bool s_validate_event_loop(struct aws_event_loop *event_loop) { return event_loop && event_loop->vtable && event_loop->impl_data; } -static bool s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loop *event_loop) { +static int s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loop *event_loop) { aws_socket->event_loop = event_loop; struct nw_socket *nw_socket = aws_socket->impl; // Never re-assign an event loop @@ -426,14 +426,14 @@ static bool s_set_event_loop(struct aws_socket *aws_socket, struct aws_event_loo "id=%p nw_socket=%p: failed to acquire event loop group.", (void *)aws_socket, (void *)nw_socket); - return false; + return AWS_OP_ERR; } nw_socket->event_loop = event_loop; AWS_LOGF_DEBUG( AWS_LS_IO_SOCKET, "id=%p nw_socket=%p: nw_socket set event loop.", (void *)aws_socket, (void *)nw_socket); - return true; + return AWS_OP_SUCCESS; } static void s_release_event_loop(struct nw_socket *nw_socket) { @@ -1807,7 +1807,7 @@ static int s_socket_connect_fn(struct aws_socket *socket, struct aws_socket_conn } /* event_loop must be set prior to setup of socket parameters. */ - if (!s_set_event_loop(socket, event_loop)) { + if (s_set_event_loop(socket, event_loop)) { goto error; } @@ -2328,7 +2328,7 @@ static int s_socket_start_accept_fn( return aws_raise_error(AWS_IO_EVENT_LOOP_ALREADY_ASSIGNED); } - if (!s_set_event_loop(socket, accept_loop)) { + if (s_set_event_loop(socket, accept_loop)) { AWS_LOGF_ERROR( AWS_LS_IO_SOCKET, "id=%p handle=%p: failed to set event loop %p, invalid event loop. It is most likely the event loop does " @@ -2455,7 +2455,7 @@ static int s_socket_assign_to_event_loop_fn(struct aws_socket *socket, struct aw socket->io_handle.data.handle, (void *)event_loop); - if (!s_set_event_loop(socket, event_loop)) { + if (s_set_event_loop(socket, event_loop)) { AWS_LOGF_ERROR( AWS_LS_IO_SOCKET, "id=%p handle=%p: assigning event loop %p failed. Invalid event loop. It is likely the event loop " From 74d6a95c4833f9f3254d62635b5608681014567a Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 30 Apr 2025 15:25:40 -0700 Subject: [PATCH 10/14] acquire event loop group in aws_channel --- source/channel.c | 6 ++++ tests/CMakeLists.txt | 1 + tests/channel_test.c | 85 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/source/channel.c b/source/channel.c index d3c5e7e55..eb4e6f0ef 100644 --- a/source/channel.c +++ b/source/channel.c @@ -257,6 +257,10 @@ struct aws_channel *aws_channel_new(struct aws_allocator *alloc, const struct aw setup_args->on_setup_completed = creation_args->on_setup_completed; setup_args->user_data = creation_args->setup_user_data; + /* keep loop alive until channel is destroyed */ + channel->loop = creation_args->event_loop; + aws_event_loop_group_acquire_from_event_loop(channel->loop); + aws_task_init(&setup_args->task, s_on_channel_setup_complete, setup_args, "on_channel_setup_complete"); aws_event_loop_schedule_task_now(creation_args->event_loop, &setup_args->task); @@ -308,6 +312,8 @@ static void s_final_channel_deletion_task(struct aws_task *task, void *arg, enum aws_channel_set_statistics_handler(channel, NULL); + aws_event_loop_group_release_from_event_loop(channel->loop); + aws_mem_release(channel->alloc, channel); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3fdb6a1be..ffb58bc4a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,6 +116,7 @@ add_test_case(channel_tasks_serialized_run) add_test_case(channel_rejects_post_shutdown_tasks) add_test_case(channel_cancels_pending_tasks) add_test_case(channel_duplicate_shutdown) +add_test_case(channel_keeps_event_loop_group_alive) add_net_test_case(channel_connect_some_hosts_timeout) add_net_test_case(test_default_with_ipv6_lookup) diff --git a/tests/channel_test.c b/tests/channel_test.c index 8fc530f99..e459c9758 100644 --- a/tests/channel_test.c +++ b/tests/channel_test.c @@ -24,6 +24,7 @@ struct channel_setup_test_args { bool setup_completed; /* protected by mutex */ bool shutdown_completed; /* protected by mutex */ int error_code; /* protected by mutex */ + bool event_loop_group_shutdown_completed; /* protected by mutex (not used by all tests) */ enum aws_task_status task_status; }; @@ -60,6 +61,19 @@ static int s_channel_setup_create_and_wait( return AWS_OP_SUCCESS; } +static void s_event_loop_group_on_shutdown_complete(void *user_data) { + struct channel_setup_test_args *setup_test_args = user_data; + aws_mutex_lock(&setup_test_args->mutex); + setup_test_args->event_loop_group_shutdown_completed = true; + aws_condition_variable_notify_all(&setup_test_args->condition_variable); + aws_mutex_unlock(&setup_test_args->mutex); +} + +static bool s_event_loop_group_shutdown_completed_predicate(void *arg) { + struct channel_setup_test_args *setup_test_args = (struct channel_setup_test_args *)arg; + return setup_test_args->event_loop_group_shutdown_completed; +} + static int s_test_channel_setup(struct aws_allocator *allocator, void *ctx) { (void)ctx; struct aws_event_loop *event_loop = aws_event_loop_new_default(allocator, aws_high_res_clock_get_ticks); @@ -845,3 +859,74 @@ static int s_test_channel_connect_some_hosts_timeout(struct aws_allocator *alloc } AWS_TEST_CASE(channel_connect_some_hosts_timeout, s_test_channel_connect_some_hosts_timeout); + +/* This is a regression test. The channel didn't used to do anything to keep the event-loop alive. + * So if the event-loop-group was released before the channel, the loops would get destroyed, + * then the channel would try to schedule its own destruction task on the loop and crash. */ +static int s_test_channel_keeps_event_loop_group_alive(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_io_library_init(allocator); + + struct channel_setup_test_args test_args = { + .mutex = AWS_MUTEX_INIT, + .condition_variable = AWS_CONDITION_VARIABLE_INIT, + }; + + struct aws_shutdown_callback_options event_loop_group_shutdown_options = { + .shutdown_callback_fn = s_event_loop_group_on_shutdown_complete, + .shutdown_callback_user_data = &test_args, + }; + struct aws_event_loop_group *event_loop_group = + aws_event_loop_group_new_default(allocator, 1, &event_loop_group_shutdown_options); + ASSERT_NOT_NULL(event_loop_group); + + struct aws_event_loop *event_loop = aws_event_loop_group_get_next_loop(event_loop_group); + + struct aws_channel_options channel_options = { + .on_setup_completed = s_channel_setup_test_on_setup_completed, + .setup_user_data = &test_args, + .on_shutdown_completed = s_channel_test_shutdown, + .shutdown_user_data = &test_args, + .event_loop = event_loop, + }; + + struct aws_channel *channel = NULL; + ASSERT_SUCCESS(s_channel_setup_create_and_wait(allocator, &channel_options, &test_args, &channel)); + + /* shut down channel, but don't clean it up yet */ + aws_channel_shutdown(channel, 0); + + ASSERT_SUCCESS(aws_mutex_lock(&test_args.mutex)); + ASSERT_SUCCESS(aws_condition_variable_wait_pred( + &test_args.condition_variable, &test_args.mutex, s_channel_test_shutdown_predicate, &test_args)); + ASSERT_SUCCESS(aws_mutex_unlock(&test_args.mutex)); + + /* release event loop group before channel */ + aws_event_loop_group_release(event_loop_group); + + /* wait a bit to ensure the event-loop-group doesn't shut down (because channel has a hold on it) */ + uint64_t wait_time = aws_timestamp_convert(500, AWS_TIMESTAMP_MILLIS, AWS_TIMESTAMP_NANOS, NULL); + ASSERT_SUCCESS(aws_mutex_lock(&test_args.mutex)); + ASSERT_FAILS( + aws_condition_variable_wait_for_pred( + &test_args.condition_variable, + &test_args.mutex, + wait_time, + s_event_loop_group_shutdown_completed_predicate, + &test_args), + "Channel failed to keep event loop alive"); + ASSERT_SUCCESS(aws_mutex_unlock(&test_args.mutex)); + + /* release channel for destruction */ + aws_channel_destroy(channel); + + /* event loop group should shut down now */ + ASSERT_SUCCESS(aws_mutex_lock(&test_args.mutex)); + ASSERT_SUCCESS(aws_condition_variable_wait_pred( + &test_args.condition_variable, &test_args.mutex, s_event_loop_group_shutdown_completed_predicate, &test_args)); + ASSERT_SUCCESS(aws_mutex_unlock(&test_args.mutex)); + + aws_io_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(channel_keeps_event_loop_group_alive, s_test_channel_keeps_event_loop_group_alive) From cfd1e49cdbe8ada54d71a2d1619c88738196660f Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 1 May 2025 09:59:29 -0700 Subject: [PATCH 11/14] print for dispatch queue schedule --- source/darwin/dispatch_queue_event_loop.c | 49 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/source/darwin/dispatch_queue_event_loop.c b/source/darwin/dispatch_queue_event_loop.c index f66a7804f..c02714468 100644 --- a/source/darwin/dispatch_queue_event_loop.c +++ b/source/darwin/dispatch_queue_event_loop.c @@ -26,6 +26,8 @@ // Start with a second for now. #define AWS_DISPATCH_QUEUE_MAX_FUTURE_SERVICE_INTERVAL (AWS_TIMESTAMP_NANOS) +uint64_t g_global_entry_index = 0; + static void s_start_destroy(struct aws_event_loop *event_loop); static void s_complete_destroy(struct aws_event_loop *event_loop); static int s_run(struct aws_event_loop *event_loop); @@ -144,6 +146,7 @@ struct scheduled_iteration_entry { uint64_t timestamp; struct aws_linked_list_node scheduled_entry_node; struct aws_dispatch_loop *dispatch_loop; + uint64_t run_index; }; /* @@ -159,6 +162,7 @@ static struct scheduled_iteration_entry *s_scheduled_iteration_entry_new( entry->allocator = dispatch_loop->allocator; entry->timestamp = timestamp; entry->dispatch_loop = s_dispatch_loop_acquire(dispatch_loop); + entry->run_index = g_global_entry_index++; return entry; } @@ -490,6 +494,17 @@ static void s_run_iteration(void *service_entry) { AWS_FATAL_ASSERT(aws_linked_list_node_is_in_list(&entry->scheduled_entry_node)); aws_linked_list_remove(&entry->scheduled_entry_node); + // run all scheduled tasks + uint64_t now_ns = 0; + aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); + dispatch_time_t dispatch_now = dispatch_time(DISPATCH_TIME_NOW, 0); + AWS_LOGF_TRACE( + AWS_LS_IO_EVENT_LOOP, + "id=%p: start run interation at %llu, dispatch time %llu, with entry %llu.", + (void *)dispatch_loop->base_loop, + now_ns, + dispatch_now, + entry->run_index); /* * If we're shutting down, then don't do anything. The destroy task handles purging and canceling tasks. @@ -530,9 +545,10 @@ static void s_run_iteration(void *service_entry) { aws_event_loop_register_tick_start(dispatch_loop->base_loop); // run all scheduled tasks - uint64_t now_ns = 0; - aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); - AWS_LOGF_TRACE(AWS_LS_IO_EVENT_LOOP, "id=%p: running scheduled tasks.", (void *)dispatch_loop->base_loop); + // uint64_t now_ns = 0; + // aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); + AWS_LOGF_TRACE( + AWS_LS_IO_EVENT_LOOP, "id=%p: running scheduled tasks at %llu.", (void *)dispatch_loop->base_loop, now_ns); aws_task_scheduler_run_all(&dispatch_loop->scheduler, now_ns); aws_event_loop_register_tick_end(dispatch_loop->base_loop); @@ -553,13 +569,19 @@ static void s_run_iteration(void *service_entry) { should_schedule = true; } /* - * If we are not scheduling a new iteration for immediate executuion, we check whether there are any tasks scheduled - * to execute now or in the future and scheudle the next iteration using that time. + * If we are not scheduling a new iteration for immediate execution, we check whether there are any tasks scheduled + * to execute now or in the future and schedule the next iteration using that time. */ else if (aws_task_scheduler_has_tasks(&dispatch_loop->scheduler, &should_schedule_at_time)) { should_schedule = true; } + AWS_LOGF_TRACE( + AWS_LS_IO_EVENT_LOOP, + "id=%p: s_run_iteration: should schedule at %llu.", + (void *)dispatch_loop->base_loop, + should_schedule_at_time); + if (should_schedule) { s_try_schedule_new_iteration(dispatch_loop, should_schedule_at_time); } @@ -636,7 +658,11 @@ static void s_try_schedule_new_iteration(struct aws_dispatch_loop *dispatch_loop */ dispatch_async_f(dispatch_loop->dispatch_queue, entry, s_run_iteration); AWS_LOGF_TRACE( - AWS_LS_IO_EVENT_LOOP, "id=%p: Scheduling run iteration on event loop.", (void *)dispatch_loop->base_loop); + AWS_LS_IO_EVENT_LOOP, + "id=%p: Scheduling run iteration %llu on event loop at %llu.", + (void *)dispatch_loop->base_loop, + entry->run_index, + now_ns); } else { /* * If the timestamp is set to execute sometime in the future, we clamp the time based on a maximum delta, @@ -646,13 +672,20 @@ static void s_try_schedule_new_iteration(struct aws_dispatch_loop *dispatch_loop * requested time. Any blocks scheduled using `dispatch_async_f()` or `dispatch_after_f()` with a closer * dispatch time will be placed on the dispatch queue and execute in order. */ + dispatch_time_t dispatch_now = dispatch_time(DISPATCH_TIME_NOW, 0); dispatch_time_t when = dispatch_time(DISPATCH_TIME_NOW, delta); + dispatch_after_f(when, dispatch_loop->dispatch_queue, entry, s_run_iteration); AWS_LOGF_TRACE( AWS_LS_IO_EVENT_LOOP, - "id=%p: Scheduling future run iteration on event loop with next occurring in %llu ns.", + "id=%p: Scheduling run iteration %llu on event loop with next occurring in %llu ns, at %llu. current " + "dispatch time: %llu, timestamp: %llu", (void *)dispatch_loop->base_loop, - delta); + entry->run_index, + delta, + when, + dispatch_now, + now_ns); } } From c6514d80612610592ca6658c5cb8df379fa5f673 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 1 May 2025 10:01:09 -0700 Subject: [PATCH 12/14] Revert "print for dispatch queue schedule" This reverts commit cfd1e49cdbe8ada54d71a2d1619c88738196660f. --- source/darwin/dispatch_queue_event_loop.c | 49 ++++------------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/source/darwin/dispatch_queue_event_loop.c b/source/darwin/dispatch_queue_event_loop.c index c02714468..f66a7804f 100644 --- a/source/darwin/dispatch_queue_event_loop.c +++ b/source/darwin/dispatch_queue_event_loop.c @@ -26,8 +26,6 @@ // Start with a second for now. #define AWS_DISPATCH_QUEUE_MAX_FUTURE_SERVICE_INTERVAL (AWS_TIMESTAMP_NANOS) -uint64_t g_global_entry_index = 0; - static void s_start_destroy(struct aws_event_loop *event_loop); static void s_complete_destroy(struct aws_event_loop *event_loop); static int s_run(struct aws_event_loop *event_loop); @@ -146,7 +144,6 @@ struct scheduled_iteration_entry { uint64_t timestamp; struct aws_linked_list_node scheduled_entry_node; struct aws_dispatch_loop *dispatch_loop; - uint64_t run_index; }; /* @@ -162,7 +159,6 @@ static struct scheduled_iteration_entry *s_scheduled_iteration_entry_new( entry->allocator = dispatch_loop->allocator; entry->timestamp = timestamp; entry->dispatch_loop = s_dispatch_loop_acquire(dispatch_loop); - entry->run_index = g_global_entry_index++; return entry; } @@ -494,17 +490,6 @@ static void s_run_iteration(void *service_entry) { AWS_FATAL_ASSERT(aws_linked_list_node_is_in_list(&entry->scheduled_entry_node)); aws_linked_list_remove(&entry->scheduled_entry_node); - // run all scheduled tasks - uint64_t now_ns = 0; - aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); - dispatch_time_t dispatch_now = dispatch_time(DISPATCH_TIME_NOW, 0); - AWS_LOGF_TRACE( - AWS_LS_IO_EVENT_LOOP, - "id=%p: start run interation at %llu, dispatch time %llu, with entry %llu.", - (void *)dispatch_loop->base_loop, - now_ns, - dispatch_now, - entry->run_index); /* * If we're shutting down, then don't do anything. The destroy task handles purging and canceling tasks. @@ -545,10 +530,9 @@ static void s_run_iteration(void *service_entry) { aws_event_loop_register_tick_start(dispatch_loop->base_loop); // run all scheduled tasks - // uint64_t now_ns = 0; - // aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); - AWS_LOGF_TRACE( - AWS_LS_IO_EVENT_LOOP, "id=%p: running scheduled tasks at %llu.", (void *)dispatch_loop->base_loop, now_ns); + uint64_t now_ns = 0; + aws_event_loop_current_clock_time(dispatch_loop->base_loop, &now_ns); + AWS_LOGF_TRACE(AWS_LS_IO_EVENT_LOOP, "id=%p: running scheduled tasks.", (void *)dispatch_loop->base_loop); aws_task_scheduler_run_all(&dispatch_loop->scheduler, now_ns); aws_event_loop_register_tick_end(dispatch_loop->base_loop); @@ -569,19 +553,13 @@ static void s_run_iteration(void *service_entry) { should_schedule = true; } /* - * If we are not scheduling a new iteration for immediate execution, we check whether there are any tasks scheduled - * to execute now or in the future and schedule the next iteration using that time. + * If we are not scheduling a new iteration for immediate executuion, we check whether there are any tasks scheduled + * to execute now or in the future and scheudle the next iteration using that time. */ else if (aws_task_scheduler_has_tasks(&dispatch_loop->scheduler, &should_schedule_at_time)) { should_schedule = true; } - AWS_LOGF_TRACE( - AWS_LS_IO_EVENT_LOOP, - "id=%p: s_run_iteration: should schedule at %llu.", - (void *)dispatch_loop->base_loop, - should_schedule_at_time); - if (should_schedule) { s_try_schedule_new_iteration(dispatch_loop, should_schedule_at_time); } @@ -658,11 +636,7 @@ static void s_try_schedule_new_iteration(struct aws_dispatch_loop *dispatch_loop */ dispatch_async_f(dispatch_loop->dispatch_queue, entry, s_run_iteration); AWS_LOGF_TRACE( - AWS_LS_IO_EVENT_LOOP, - "id=%p: Scheduling run iteration %llu on event loop at %llu.", - (void *)dispatch_loop->base_loop, - entry->run_index, - now_ns); + AWS_LS_IO_EVENT_LOOP, "id=%p: Scheduling run iteration on event loop.", (void *)dispatch_loop->base_loop); } else { /* * If the timestamp is set to execute sometime in the future, we clamp the time based on a maximum delta, @@ -672,20 +646,13 @@ static void s_try_schedule_new_iteration(struct aws_dispatch_loop *dispatch_loop * requested time. Any blocks scheduled using `dispatch_async_f()` or `dispatch_after_f()` with a closer * dispatch time will be placed on the dispatch queue and execute in order. */ - dispatch_time_t dispatch_now = dispatch_time(DISPATCH_TIME_NOW, 0); dispatch_time_t when = dispatch_time(DISPATCH_TIME_NOW, delta); - dispatch_after_f(when, dispatch_loop->dispatch_queue, entry, s_run_iteration); AWS_LOGF_TRACE( AWS_LS_IO_EVENT_LOOP, - "id=%p: Scheduling run iteration %llu on event loop with next occurring in %llu ns, at %llu. current " - "dispatch time: %llu, timestamp: %llu", + "id=%p: Scheduling future run iteration on event loop with next occurring in %llu ns.", (void *)dispatch_loop->base_loop, - entry->run_index, - delta, - when, - dispatch_now, - now_ns); + delta); } } From 7f3ed340db20c4599daca8c06aad084037397127 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 5 May 2025 10:05:24 -0700 Subject: [PATCH 13/14] update checksum proof-alarm&clang-format --- .github/workflows/proof-alarm.yml | 2 +- tests/channel_test.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/proof-alarm.yml b/.github/workflows/proof-alarm.yml index e44160ed4..9faa2f2fc 100644 --- a/.github/workflows/proof-alarm.yml +++ b/.github/workflows/proof-alarm.yml @@ -16,7 +16,7 @@ jobs: - name: Check run: | TMPFILE=$(mktemp) - echo "844fcb2768afd3ab013fa49447c1b855 source/linux/epoll_event_loop.c" > $TMPFILE + echo "5109c3b2748a98621ebdc05756fdfa51 source/linux/epoll_event_loop.c" > $TMPFILE md5sum --check $TMPFILE # No further steps if successful diff --git a/tests/channel_test.c b/tests/channel_test.c index e459c9758..93dc7f4a5 100644 --- a/tests/channel_test.c +++ b/tests/channel_test.c @@ -21,9 +21,9 @@ struct channel_setup_test_args { struct aws_mutex mutex; struct aws_condition_variable condition_variable; - bool setup_completed; /* protected by mutex */ - bool shutdown_completed; /* protected by mutex */ - int error_code; /* protected by mutex */ + bool setup_completed; /* protected by mutex */ + bool shutdown_completed; /* protected by mutex */ + int error_code; /* protected by mutex */ bool event_loop_group_shutdown_completed; /* protected by mutex (not used by all tests) */ enum aws_task_status task_status; }; From 6c51a190867466e8d97a376508bcc5323ce62757 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 5 May 2025 10:19:29 -0700 Subject: [PATCH 14/14] add null check --- source/event_loop.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/event_loop.c b/source/event_loop.c index 88ce5e21b..50e9c710e 100644 --- a/source/event_loop.c +++ b/source/event_loop.c @@ -381,13 +381,16 @@ void aws_event_loop_group_release(struct aws_event_loop_group *el_group) { } struct aws_event_loop_group *aws_event_loop_group_acquire_from_event_loop(struct aws_event_loop *event_loop) { - AWS_ASSERT(event_loop); - return aws_event_loop_group_acquire(event_loop->base_elg); + if (event_loop != NULL) { + return aws_event_loop_group_acquire(event_loop->base_elg); + } + return NULL; } void aws_event_loop_group_release_from_event_loop(struct aws_event_loop *event_loop) { - AWS_ASSERT(event_loop); - aws_event_loop_group_release(event_loop->base_elg); + if (event_loop != NULL) { + aws_event_loop_group_release(event_loop->base_elg); + } } size_t aws_event_loop_group_get_loop_count(const struct aws_event_loop_group *el_group) {