Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@
#pragma once
#include "tmc/detail/impl.hpp" // IWYU pragma: keep

// Common implementations of .result_each() awaitable functions
// for spawn_many and spawn_tuple

#include "tmc/ex_any.hpp"
// Common implementations of awaitable functions for mux_many and mux_tuple

#include <atomic>
#include <coroutine>
Expand All @@ -18,18 +15,20 @@
namespace tmc {
namespace detail {

TMC_DECL bool result_each_await_suspend(
TMC_DECL bool mux_await_suspend(
ptrdiff_t remaining_count, std::coroutine_handle<> Outer,
std::coroutine_handle<>& continuation, std::atomic<size_t>& sync_flags
) noexcept;

TMC_DECL size_t result_each_await_resume(
ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags
) noexcept;
TMC_DECL size_t
mux_await_resume(ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags) noexcept;

TMC_DECL size_t
mux_poll(ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags) noexcept;

} // namespace detail
} // namespace tmc

#if !defined(TMC_STANDALONE_COMPILATION) || defined(TMC_IMPL)
#include "tmc/detail/result_each.ipp"
#include "tmc/detail/mux_shared.ipp"
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
#pragma once

#include "tmc/detail/awaitable_customizer.hpp"
#include "tmc/detail/compat.hpp"
#include "tmc/detail/impl.hpp" // IWYU pragma: keep
#include "tmc/detail/result_each.hpp"
#include "tmc/detail/mux_shared.hpp"

#include <atomic>
#include <bit>
Expand All @@ -17,7 +18,7 @@
namespace tmc {
namespace detail {

bool result_each_await_suspend(
bool mux_await_suspend(
ptrdiff_t remaining_count, std::coroutine_handle<> Outer,
std::coroutine_handle<>& continuation, std::atomic<size_t>& sync_flags
) noexcept {
Expand Down Expand Up @@ -50,11 +51,10 @@ bool result_each_await_suspend(
}
}

size_t result_each_await_resume(
ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags
) noexcept {
size_t
mux_await_resume(ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags) noexcept {
if (remaining_count == 0) {
return 64;
return TMC_PLATFORM_BITS;
}
size_t resumeState = sync_flags.load(std::memory_order_acquire);
assert((resumeState & tmc::detail::task_flags::EACH) != 0);
Expand All @@ -67,5 +67,28 @@ size_t result_each_await_resume(
return slot;
}

size_t mux_poll(ptrdiff_t& remaining_count, std::atomic<size_t>& sync_flags) noexcept {
if (remaining_count == 0) {
// No submitted results remain: same terminal value as await_resume(),
// which the caller compares against end().
return TMC_PLATFORM_BITS;
}
// The owner holds the EACH (lock) bit while it is running, so children that
// complete concurrently set their result bit but cannot resume us. The
// acquire load synchronizes-with the child's release-store of that bit,
// making its result visible.
size_t state = sync_flags.load(std::memory_order_acquire);
assert((state & tmc::detail::task_flags::EACH) != 0);
size_t slots = state & ~tmc::detail::task_flags::EACH;
if (slots == 0) {
// Results are still pending, but none is ready yet.
return TMC_PLATFORM_BITS + 1;
}
size_t slot = static_cast<size_t>(std::countr_zero(slots));
--remaining_count;
sync_flags.fetch_sub(TMC_ONE_BIT << slot, std::memory_order_release);
return slot;
}

} // namespace detail
} // namespace tmc
27 changes: 22 additions & 5 deletions include/tmc/mux_many.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "tmc/detail/compat.hpp"
#include "tmc/detail/concepts_awaitable.hpp" // IWYU pragma: keep
#include "tmc/detail/concepts_work_item.hpp"
#include "tmc/detail/result_each.hpp"
#include "tmc/detail/mux_shared.hpp"
#include "tmc/detail/thread_locals.hpp"
#include "tmc/detail/tsan.hpp"
#include "tmc/ex_any.hpp"
Expand Down Expand Up @@ -308,7 +308,7 @@ class mux_many : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
// mux_many can be awaited in a different context than where it was
// created. Therefore, capture the continuation_executor at the await point.
continuation_executor = tmc::detail::this_thread::executor();
return tmc::detail::result_each_await_suspend(
return tmc::detail::mux_await_suspend(
remaining_count, Outer, continuation, sync_flags
);
}
Expand All @@ -320,7 +320,7 @@ class mux_many : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
/// submission. When no submitted results remain, the index returned will be
/// equal to the value of `end()`.
[[nodiscard]] inline size_t await_resume() noexcept {
auto slot = tmc::detail::result_each_await_resume(remaining_count, sync_flags);
auto slot = tmc::detail::mux_await_resume(remaining_count, sync_flags);
if (slot != end()) {
active_slots &= ~(TMC_ONE_BIT << slot);
}
Expand All @@ -331,9 +331,26 @@ class mux_many : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
/// movable). The awaiter is the group itself.
mux_many& operator co_await() & noexcept { return *this; }

/// Non-suspending check for a ready result. There are three outcomes:
/// - a result is ready: it is consumed and its index returned (like `co_await`).
/// - results are still pending but none are ready: returns `pending()`.
/// - no submitted results remain: returns `end()`.
[[nodiscard]] inline size_t poll() noexcept {
auto slot = tmc::detail::mux_poll(remaining_count, sync_flags);
if (slot < TMC_PLATFORM_BITS) {
active_slots &= ~(TMC_ONE_BIT << slot);
}
return slot;
}

/// Provides a sentinel value that can be compared against the value returned
/// from co_await.
inline constexpr size_t end() const noexcept { return 64; }
/// from `co_await` or `poll()`.
inline constexpr size_t end() const noexcept { return TMC_PLATFORM_BITS; }

/// Provides a sentinel value returned by `poll()` to indicate that no result is ready
/// right now, but will become ready later. This is distinct from `end()`, which
/// indicates that all submitted results have been consumed.
inline constexpr size_t pending() const noexcept { return TMC_PLATFORM_BITS + 1; }

/// Returns the capacity of the mux, equal to the `Count` template argument.
/// This is the maximum number of awaitables that may be active concurrently.
Expand Down
27 changes: 22 additions & 5 deletions include/tmc/mux_tuple.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "tmc/detail/compat.hpp"
#include "tmc/detail/concepts_awaitable.hpp" // IWYU pragma: keep
#include "tmc/detail/concepts_work_item.hpp"
#include "tmc/detail/result_each.hpp"
#include "tmc/detail/mux_shared.hpp"
#include "tmc/detail/thread_locals.hpp"
#include "tmc/detail/tsan.hpp"
#include "tmc/detail/tuple_helpers.hpp"
Expand Down Expand Up @@ -201,7 +201,7 @@ class mux_tuple : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
// mux_tuple can be awaited in a different context than where it was
// created. Therefore, capture the continuation_executor at the await point.
continuation_executor = tmc::detail::this_thread::executor();
return tmc::detail::result_each_await_suspend(
return tmc::detail::mux_await_suspend(
remaining_count, Outer, continuation, sync_flags
);
}
Expand All @@ -211,7 +211,7 @@ class mux_tuple : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
/// exactly once per submission. When no submitted results remain, the index
/// returned will be equal to the value of `end()`.
[[nodiscard]] inline size_t await_resume() noexcept {
auto slot = tmc::detail::result_each_await_resume(remaining_count, sync_flags);
auto slot = tmc::detail::mux_await_resume(remaining_count, sync_flags);
if (slot != end()) {
active_slots &= ~(TMC_ONE_BIT << slot);
}
Expand All @@ -222,9 +222,26 @@ class mux_tuple : private tmc::detail::AwaitTagNoGroupCoAwaitLvalue {
/// movable). The awaiter is the group itself.
mux_tuple& operator co_await() & noexcept { return *this; }

/// Non-suspending check for a ready result. There are three outcomes:
/// - a result is ready: it is consumed and its index returned (like `co_await`).
/// - results are still pending but none are ready: returns `pending()`.
/// - no submitted results remain: returns `end()`.
[[nodiscard]] inline size_t poll() noexcept {
auto slot = tmc::detail::mux_poll(remaining_count, sync_flags);
if (slot < TMC_PLATFORM_BITS) {
active_slots &= ~(TMC_ONE_BIT << slot);
}
return slot;
}

/// Provides a sentinel value that can be compared against the value returned
/// from co_await.
inline constexpr size_t end() const noexcept { return 64; }
/// from `co_await` or `poll()`.
inline constexpr size_t end() const noexcept { return TMC_PLATFORM_BITS; }

/// Provides a sentinel value returned by `poll()` to indicate that no result is ready
/// right now, but will become ready later. This is distinct from `end()`, which
/// indicates that all submitted results have been consumed.
inline constexpr size_t pending() const noexcept { return TMC_PLATFORM_BITS + 1; }

/// Returns the capacity of the mux, equal to the number of `Result` template arguments.
/// This is the maximum number of awaitables that may be active concurrently.
Expand Down
Loading