|
10 | 10 | #include "tmc/detail/thread_locals.hpp" |
11 | 11 | #include "tmc/detail/waiter_list.hpp" |
12 | 12 |
|
| 13 | +#include <array> |
13 | 14 | #include <atomic> |
14 | 15 | #include <cassert> |
15 | 16 | #include <coroutine> |
@@ -125,15 +126,56 @@ void waiter_list::configure_and_add_waiter( |
125 | 126 | )); |
126 | 127 | } |
127 | 128 |
|
128 | | -void waiter_list::wake_all() noexcept { |
129 | | - auto curr = take_all(); |
130 | | - while (curr != nullptr) { |
131 | | - auto next = curr->next; |
132 | | - curr->waiter.resume(); |
133 | | - curr = next; |
| 129 | +size_t wake_waiters_in_batches(waiter_list_node* Curr) noexcept { |
| 130 | + std::array<tmc::work_item, 64> batch; |
| 131 | + size_t batchSize = 0; |
| 132 | + tmc::ex_any* continuationExecutor = nullptr; |
| 133 | + size_t continuationPriority = 0; |
| 134 | + size_t wakeCount = 0; |
| 135 | + |
| 136 | + while (Curr != nullptr) { |
| 137 | + auto next = Curr->next; |
| 138 | + |
| 139 | + // A batch can only be posted to a single executor at a single priority. |
| 140 | + // If this waiter's executor or priority differs from the batch in |
| 141 | + // progress, post that batch before starting a new one for this waiter. |
| 142 | + if (batchSize != 0 && (Curr->waiter.continuation_executor != continuationExecutor || |
| 143 | + Curr->waiter.continuation_priority != continuationPriority)) { |
| 144 | + tmc::detail::post_bulk_checked( |
| 145 | + continuationExecutor, batch.data(), batchSize, continuationPriority |
| 146 | + ); |
| 147 | + batchSize = 0; |
| 148 | + } |
| 149 | + |
| 150 | + if (batchSize == 0) { |
| 151 | + continuationExecutor = Curr->waiter.continuation_executor; |
| 152 | + continuationPriority = Curr->waiter.continuation_priority; |
| 153 | + } |
| 154 | + |
| 155 | + batch[batchSize] = Curr->waiter.continuation; |
| 156 | + ++batchSize; |
| 157 | + ++wakeCount; |
| 158 | + if (batchSize == batch.size()) { |
| 159 | + tmc::detail::post_bulk_checked( |
| 160 | + continuationExecutor, batch.data(), batchSize, continuationPriority |
| 161 | + ); |
| 162 | + batchSize = 0; |
| 163 | + } |
| 164 | + |
| 165 | + Curr = next; |
| 166 | + } |
| 167 | + |
| 168 | + if (batchSize != 0) { |
| 169 | + tmc::detail::post_bulk_checked( |
| 170 | + continuationExecutor, batch.data(), batchSize, continuationPriority |
| 171 | + ); |
134 | 172 | } |
| 173 | + |
| 174 | + return wakeCount; |
135 | 175 | } |
136 | 176 |
|
| 177 | +void waiter_list::wake_all() noexcept { wake_waiters_in_batches(take_all()); } |
| 178 | + |
137 | 179 | waiter_list_node* waiter_list::take_all() noexcept { |
138 | 180 | // Drain the input stack. take_all is only used when every waiter will be |
139 | 181 | // woken, so the order doesn't matter; no need to reverse. |
|
0 commit comments