Skip to content

Commit

Permalink
Merge branch 'main' into bug/detect_card_in_lazy_import
Browse files Browse the repository at this point in the history
  • Loading branch information
mergify[bot] authored Aug 2, 2023
2 parents 1b59fdc + 7943b78 commit 3ebaa68
Show file tree
Hide file tree
Showing 23 changed files with 1,533 additions and 38 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/python.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ jobs:
with:
fetch-depth: 0
submodules: recursive

- name: Add msbuild to PATH
if: ${{ matrix.os == 'windows-latest'}}
uses: microsoft/[email protected]
- name: Set up conda ${{ matrix.python-version }}
uses: conda-incubator/setup-miniconda@v2
if: ${{ matrix.module != 'gpu' }}
Expand Down
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,7 @@ Makefile
cpp/collective/gloo/include/config.h

#filestore
python/xoscar/collective/tests/collective
python/xoscar/collective/tests/collective

#libuv
python/xoscar/collective/uv.dll
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@
[submodule "third_party/gloo"]
path = third_party/gloo
url = https://github.com/facebookincubator/gloo.git
[submodule "third_party/libuv"]
path = third_party/libuv
url = https://github.com/libuv/libuv.git
30 changes: 27 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
cmake_minimum_required(VERSION 3.11...3.21)

project(XoscarCollective)

if(NOT DEFINED PYTHON_PATH)
find_package(Python COMPONENTS Interpreter Development)
else()
Expand All @@ -24,6 +23,23 @@ execute_process(COMMAND python -c "import sysconfig; print(sysconfig.get_path('i

# Set include directories
include_directories(${PYTHON_INCLUDE_PATH})
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(libuv_dir ${CMAKE_SOURCE_DIR}/third_party/libuv/build/uvlib)
if(NOT EXISTS ${libuv_dir})
execute_process(
COMMAND
cmd /c
"cd ..\\..\\..\\..\\..\\third_party\\libuv&&mkdir build&&cd build && mkdir uvlib&&cmake .. -DCMAKE_INSTALL_PREFIX=uvlib&&msbuild.exe INSTALL.vcxproj"
)
endif()
set(libuv_ROOT ${CMAKE_SOURCE_DIR}/third_party/libuv/build/uvlib)
set(uv_HEADER_PATH ${CMAKE_SOURCE_DIR}/third_party/libuv/include)
include_directories(${uv_HEADER_PATH})
#copy uv.dll to /python/xoscar/collective
file(COPY ${CMAKE_SOURCE_DIR}/third_party/libuv/build/uvlib/bin/uv.dll
DESTINATION ${CMAKE_SOURCE_DIR}/python/xoscar/collective)
add_definitions(-DNOMINMAX)
endif()

add_subdirectory(third_party/fmt)
add_subdirectory(third_party/pybind11)
Expand All @@ -35,8 +51,16 @@ set_target_properties(
PROPERTIES CXX_STANDARD 11
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF)

if(NOT DEFINED BUILD_TMP_DIR)
file(GLOB TMP_DIRS "python/build/temp*")
foreach(TMP_DIR ${TMP_DIRS})
set(BUILD_TMP_DIR ${TMP_DIR}/xoscar_pygloo)
endforeach()
else()
set(BUILD_TMP_DIR python/${BUILD_TMP_DIR})
endif()
# copy config.h to cpp/gloo/include
file(COPY python/${BUILD_TMP_DIR}/third_party/gloo/gloo/config.h
file(COPY ${BUILD_TMP_DIR}/third_party/gloo/gloo/config.h
DESTINATION ${CMAKE_SOURCE_DIR}/cpp/collective/gloo/include)

add_subdirectory(cpp)
1 change: 1 addition & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ add_subdirectory(collective/rendezvous)
add_subdirectory(collective/gloo)

pybind11_add_module(xoscar_pygloo collective/gloo/main.cc)

target_link_libraries(xoscar_pygloo PRIVATE GlooLib gloo StoreLib fmt::fmt)
set_target_properties(xoscar_pygloo PROPERTIES LIBRARY_OUTPUT_DIRECTORY
${LIBRARY_OUTPUT_DIRECTORY})
11 changes: 10 additions & 1 deletion cpp/collective/rendezvous/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,22 @@ project(
VERSION 0.0.1
LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 20)
# there will be an error that
# "Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.36.32532\include\format"
# on windows if c++ standard is set to 20
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(CMAKE_CXX_STANDARD 17)
else()
set(CMAKE_CXX_STANDARD 20)
endif()

include_directories(include)

add_library(
StoreLib
include/error.h
include/call_once.h
include/win_sock_utils.hpp
include/exception.h
src/exception.cpp
include/socket.h
Expand Down
110 changes: 110 additions & 0 deletions cpp/collective/rendezvous/include/call_once.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/* Copyright 2022-2023 XProbe Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#pragma once

#include <atomic>
#include <functional>
#include <memory>
#include <mutex>
#include <thread>
#include <utility>

#if defined(__GNUC__) || defined(__ICL) || defined(__clang__)
# ifndef XOSCAR_LIKELY
# define XOSCAR_LIKELY(expr) \
(__builtin_expect(static_cast<bool>(expr), 1))
# define XOSCAR_UNLIKELY(expr) \
(__builtin_expect(static_cast<bool>(expr), 0))
# endif
#else
# ifndef XOSCAR_LIKELY
# define XOSCAR_LIKELY(expr) (expr)
# define XOSCAR_UNLIKELY(expr) (expr)
# endif
#endif

namespace xoscar {

template <typename F, typename... args>
#if defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable >= 201703L
using invoke_result = typename std::invoke_result<F, args...>;
#else
using invoke_result = typename std::result_of<F && (args && ...)>;
#endif

template <typename F, typename... args>
using invoke_result_t = typename invoke_result<F, args...>::type;

template <typename Functor, typename... Args>
typename std::enable_if<
std::is_member_pointer<typename std::decay<Functor>::type>::value,
typename xoscar::invoke_result_t<Functor, Args...>>::type
invoke(Functor &&f, Args &&...args) {
return std::mem_fn(std::forward<Functor>(f))(std::forward<Args>(args)...);
}

template <typename Functor, typename... Args>
typename std::enable_if<
!std::is_member_pointer<typename std::decay<Functor>::type>::value,
typename xoscar::invoke_result_t<Functor, Args...>>::type
invoke(Functor &&f, Args &&...args) {
return std::forward<Functor>(f)(std::forward<Args>(args)...);
}

// custom xoscar call_once implementation to avoid the deadlock in
// std::call_once. The implementation here is a simplified version from folly
// and likely much much higher memory footprint.
template <typename Flag, typename F, typename... Args>
inline void call_once(Flag &flag, F &&f, Args &&...args) {
if (XOSCAR_LIKELY(flag.test_once())) {
return;
}
flag.call_once_slow(std::forward<F>(f), std::forward<Args>(args)...);
}

class once_flag {
public:
#ifndef _WIN32
constexpr
#endif

once_flag() noexcept = default;
once_flag(const once_flag &) = delete;
once_flag &operator=(const once_flag &) = delete;

private:
template <typename Flag, typename F, typename... Args>
friend void call_once(Flag &flag, F &&f, Args &&...args);

template <typename F, typename... Args>
void call_once_slow(F &&f, Args &&...args) {
std::lock_guard<std::mutex> guard(mutex_);
if (init_.load(std::memory_order_relaxed)) {
return;
}
invoke(f, std::forward<Args>(args)...);
init_.store(true, std::memory_order_release);
}

bool test_once() { return init_.load(std::memory_order_acquire); }

void reset_once() { init_.store(false, std::memory_order_release); }

private:
std::mutex mutex_;
std::atomic<bool> init_{false};
};

} // namespace xoscar
3 changes: 3 additions & 0 deletions cpp/collective/rendezvous/include/socket.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ limitations under the License. */
#include <chrono>
#include <cstdint>
#include <ctime>
#include <iostream>
#include <memory>
#include <string>

#define XOSCAR_WARNING(...)

namespace xoscar {
namespace detail {

Expand Down
Loading

0 comments on commit 3ebaa68

Please sign in to comment.