Skip to content

Commit bec7503

Browse files
authored
barrier and rw_lock wake waiters in batches of 64 (#272)
1 parent 5e3d4a9 commit bec7503

5 files changed

Lines changed: 71 additions & 74 deletions

File tree

‎include/tmc/detail/barrier.ipp‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,18 @@ bool aw_barrier::await_suspend(std::coroutine_handle<> Outer) noexcept {
3939
// Reset this
4040
parent.done_count = parent.start_count.load();
4141

42-
// Resume the waiters
43-
while (curr != nullptr) {
44-
auto next = curr->next;
45-
if (curr != &me) {
46-
// Symmetric transfer to this coroutine.
47-
// Others are posted to the executor.
48-
curr->waiter.resume();
49-
}
50-
curr = next;
42+
// This coroutine resumes by symmetric transfer (returning false), so unlink
43+
// `me` from the chain before waking the others in batches. `me` is
44+
// guaranteed to be in the chain since it was added above.
45+
tmc::detail::waiter_list_node head;
46+
head.next = curr;
47+
auto prev = &head;
48+
while (prev->next != &me) {
49+
prev = prev->next;
5150
}
51+
prev->next = me.next;
52+
53+
tmc::detail::wake_waiters_in_batches(head.next);
5254
return false;
5355
}
5456

‎include/tmc/detail/manual_reset_event.ipp‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -11,58 +11,10 @@
1111
#include "tmc/detail/waiter_list.hpp"
1212
#include "tmc/manual_reset_event.hpp"
1313

14-
#include <array>
1514
#include <atomic>
1615
#include <coroutine>
1716

1817
namespace tmc {
19-
namespace detail {
20-
// Submits a chain of waiters to their continuation executors, grouping
21-
// consecutive waiters that share an executor and priority into bulk posts.
22-
// Returns the number of waiters woken.
23-
inline size_t wake_waiters_in_batches(tmc::detail::waiter_list_node* curr) noexcept {
24-
std::array<tmc::work_item, 64> batch;
25-
size_t batchSize = 0;
26-
tmc::ex_any* continuationExecutor = nullptr;
27-
size_t continuationPriority = 0;
28-
size_t wakeCount = 0;
29-
30-
while (curr != nullptr) {
31-
auto next = curr->next;
32-
33-
// A batch can only be posted to a single executor at a single priority.
34-
// If this waiter's executor or priority differs from the batch in
35-
// progress, post that batch before starting a new one for this waiter.
36-
if (batchSize != 0 && (curr->waiter.continuation_executor != continuationExecutor ||
37-
curr->waiter.continuation_priority != continuationPriority)) {
38-
continuationExecutor->post_bulk(batch.data(), batchSize, continuationPriority);
39-
batchSize = 0;
40-
}
41-
42-
if (batchSize == 0) {
43-
continuationExecutor = curr->waiter.continuation_executor;
44-
continuationPriority = curr->waiter.continuation_priority;
45-
}
46-
47-
batch[batchSize] = curr->waiter.continuation;
48-
++batchSize;
49-
++wakeCount;
50-
if (batchSize == batch.size()) {
51-
continuationExecutor->post_bulk(batch.data(), batchSize, continuationPriority);
52-
batchSize = 0;
53-
}
54-
55-
curr = next;
56-
}
57-
58-
if (batchSize != 0) {
59-
continuationExecutor->post_bulk(batch.data(), batchSize, continuationPriority);
60-
}
61-
62-
return wakeCount;
63-
}
64-
} // namespace detail
65-
6618
bool aw_manual_reset_event::await_suspend(std::coroutine_handle<> Outer) noexcept {
6719
me.waiter.continuation = Outer;
6820
me.waiter.continuation_executor = tmc::detail::this_thread::executor();

‎include/tmc/detail/rw_lock.ipp‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,12 +180,7 @@ template <bool PreferWriters> void rw_lock::try_wake(state_t V) noexcept {
180180
}
181181
wakeTail->next = nullptr;
182182

183-
auto toWake = wakeHead.next;
184-
while (toWake != nullptr) {
185-
auto next = toWake->next;
186-
toWake->waiter.resume();
187-
toWake = next;
188-
}
183+
tmc::detail::wake_waiters_in_batches(wakeHead.next);
189184
return;
190185
}
191186
}

‎include/tmc/detail/waiter_list.hpp‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ struct waiter_list_node {
6060
suspend(waiter_data_base* Parent, std::coroutine_handle<> Outer) noexcept;
6161
};
6262

63+
/// Submits a chain of waiters to their continuation executors, grouping
64+
/// consecutive waiters that share an executor and priority into bulk posts of
65+
/// up to 64 at a time. Returns the number of waiters woken.
66+
TMC_DECL size_t wake_waiters_in_batches(waiter_list_node* Curr) noexcept;
67+
6368
/// Two-stack waiter queue with FIFO wake order.
6469
///
6570
/// `input` is a lock-free Treiber stack that producers push to via
@@ -84,11 +89,12 @@ class waiter_list {
8489
waiter_list_node& Node, std::coroutine_handle<> Outer
8590
) noexcept;
8691

87-
/// Wakes all waiters. Not guaranteed to wake in FIFO order (which doesn't
88-
/// matter since waking all waiters doesn't guarantee they get *processed* in
89-
/// FIFO order either after going through their executor queues). Both lists
90-
/// become empty. Thread-safe with concurrent `configure_and_add_waiter`, but
91-
/// not with concurrent consumers (`maybe_wake`, `must_take_1`, `take_all`).
92+
/// Wakes all waiters, posting them to their executors in batches. Not
93+
/// guaranteed to wake in FIFO order (which doesn't matter since waking all
94+
/// waiters doesn't guarantee they get *processed* in FIFO order either after
95+
/// going through their executor queues). Both lists become empty.
96+
/// Thread-safe with concurrent `configure_and_add_waiter`, but not with
97+
/// concurrent consumers (`maybe_wake`, `must_take_1`, `take_all`).
9298
TMC_DECL void wake_all() noexcept;
9399

94100
/// Returns the number of waiters currently in the list (both `output` and

‎include/tmc/detail/waiter_list.ipp‎

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "tmc/detail/thread_locals.hpp"
1111
#include "tmc/detail/waiter_list.hpp"
1212

13+
#include <array>
1314
#include <atomic>
1415
#include <cassert>
1516
#include <coroutine>
@@ -125,15 +126,56 @@ void waiter_list::configure_and_add_waiter(
125126
));
126127
}
127128

128-
void waiter_list::wake_all() noexcept {
129-
auto curr = take_all();
130-
while (curr != nullptr) {
131-
auto next = curr->next;
132-
curr->waiter.resume();
133-
curr = next;
129+
size_t wake_waiters_in_batches(waiter_list_node* Curr) noexcept {
130+
std::array<tmc::work_item, 64> batch;
131+
size_t batchSize = 0;
132+
tmc::ex_any* continuationExecutor = nullptr;
133+
size_t continuationPriority = 0;
134+
size_t wakeCount = 0;
135+
136+
while (Curr != nullptr) {
137+
auto next = Curr->next;
138+
139+
// A batch can only be posted to a single executor at a single priority.
140+
// If this waiter's executor or priority differs from the batch in
141+
// progress, post that batch before starting a new one for this waiter.
142+
if (batchSize != 0 && (Curr->waiter.continuation_executor != continuationExecutor ||
143+
Curr->waiter.continuation_priority != continuationPriority)) {
144+
tmc::detail::post_bulk_checked(
145+
continuationExecutor, batch.data(), batchSize, continuationPriority
146+
);
147+
batchSize = 0;
148+
}
149+
150+
if (batchSize == 0) {
151+
continuationExecutor = Curr->waiter.continuation_executor;
152+
continuationPriority = Curr->waiter.continuation_priority;
153+
}
154+
155+
batch[batchSize] = Curr->waiter.continuation;
156+
++batchSize;
157+
++wakeCount;
158+
if (batchSize == batch.size()) {
159+
tmc::detail::post_bulk_checked(
160+
continuationExecutor, batch.data(), batchSize, continuationPriority
161+
);
162+
batchSize = 0;
163+
}
164+
165+
Curr = next;
166+
}
167+
168+
if (batchSize != 0) {
169+
tmc::detail::post_bulk_checked(
170+
continuationExecutor, batch.data(), batchSize, continuationPriority
171+
);
134172
}
173+
174+
return wakeCount;
135175
}
136176

177+
void waiter_list::wake_all() noexcept { wake_waiters_in_batches(take_all()); }
178+
137179
waiter_list_node* waiter_list::take_all() noexcept {
138180
// Drain the input stack. take_all is only used when every waiter will be
139181
// woken, so the order doesn't matter; no need to reverse.

0 commit comments

Comments
 (0)