Merge remote-tracking branch 'upstream/main'

# Conflicts: # .env # VERSION # bin/workon.sh # cmake/ExternalProjects.cmake # docker/faabric-base.dockerfile # docker/faabric.dockerfile # include/faabric/endpoint/FaabricEndpoint.h # include/faabric/endpoint/FaabricEndpointHandler.h # src/endpoint/FaabricEndpoint.cpp # src/endpoint/FaabricEndpointHandler.cpp # src/scheduler/Scheduler.cpp # src/transport/MessageEndpoint.cpp # tasks/docker.py # tests/test/endpoint/test_handler.cpp
auto-ndp · Oct 13, 2022 · 4cbc90c · 4cbc90c
2 parents 578ca87 + d686b8e
commit 4cbc90c
Show file tree

Hide file tree

Showing 25 changed files with 357 additions and 203 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -9,57 +9,55 @@ on:
 
 defaults:
   run:
+    working-directory: /code/faabric
     shell: bash
 
 jobs:
   conan-cache:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.3
-    defaults:
-      run:
-        working-directory: /code/faabric
+      image: faasm/faabric:0.3.5
     steps:
       - uses: faasm/conan-cache-action@v1
         with:
            directory: /code/faabric
       - name: "Build dependencies to be shared by all runs"
-        run: inv dev.cmake -b Debug
+        run: ./bin/inv_wrapper.sh dev.cmake -b Debug
 
   docs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.3
-    defaults:
-      run:
-        working-directory: /code/faabric
+      image: faasm/faabric:0.3.5
     steps:
       - name: "Fetch ref"
         run: git fetch origin ${GITHUB_REF}:ci-branch
       - name: "Check out branch"
         run: git checkout --force ci-branch
       - name: "Build docs"
-        run: inv docs
+        run: ./bin/inv_wrapper.sh docs
 
   formatting:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/faabric:0.3.3
-    defaults:
-      run:
-        working-directory: /code/faabric
+      image: faasm/faabric:0.3.5
     steps:
       # --- Code update ---
       - name: "Fetch ref"
         run: git fetch origin ${GITHUB_REF}:ci-branch
       - name: "Check out branch"
         run: git checkout --force ci-branch
+      - name: "Update python deps"
+        run: |
+          source venv/bin/activate
+          pip3 install -r requirements.txt
       # --- Formatting checks ---
       - name: "Python formatting check"
-        run: ./bin/check_python.sh
+        run: |
+          source venv/bin/activate
+          ./bin/check_python.sh
       - name: "Run C/C++ formatting"
         run: ./bin/run_clang_format.sh
       - name: "Check C/C++ formatting changes"
@@ -81,11 +79,8 @@ jobs:
       TSAN_OPTIONS: "verbosity=1:halt_on_error=1:suppressions=/code/faabric/thread-sanitizer-ignorelist.txt:history_size=7"
       UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
     container:
-      image: faasm/faabric:0.3.3
+      image: faasm/faabric:0.3.5
       options: --privileged
-    defaults:
-      run:
-        working-directory: /code/faabric
     services:
       redis:
         image: redis
@@ -96,9 +91,9 @@ jobs:
       - name: "Ping redis"
         run: redis-cli -h redis ping
       - name: "Run cmake for tests"
-        run: inv dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }}
+        run: ./bin/inv_wrapper.sh dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }}
       - name: "Build tests"
-        run: inv dev.cc faabric_tests
+        run: ./bin/inv_wrapper.sh dev.cc faabric_tests
       - name: "Run tests"
         run: ./bin/faabric_tests
         working-directory: /build/faabric/static
@@ -109,6 +104,9 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CONAN_CACHE_MOUNT_SOURCE: ~/.conan/
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
     steps:
       # --- Code update ---
       - name: "Check out code"
@@ -131,10 +129,7 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: faasm/faabric:0.3.3
-    defaults:
-      run:
-        working-directory: /code/faabric
+      image: faasm/faabric:0.3.5
     services:
       redis:
         image: redis
@@ -144,12 +139,12 @@ jobs:
           build-type: "release"
           directory: /code/faabric
       - name: "Run cmake shared"
-        run: inv dev.cmake --shared --build=Release
+        run: ./bin/inv_wrapper.sh dev.cmake --shared --build=Release
       - name: "Build Faabric shared library"
-        run: inv dev.cc faabric --shared
+        run: ./bin/inv_wrapper.sh dev.cc faabric --shared
       - name: "Install Faabric shared library"
-        run: inv dev.install faabric --shared
+        run: ./bin/inv_wrapper.sh dev.install faabric --shared
       - name: "Build examples"
-        run: inv examples
+        run: ./bin/inv_wrapper.sh examples
       - name: "Run example to check"
-        run: inv examples.execute check
+        run: ./bin/inv_wrapper.sh examples.execute check
diff --git a/.gitignore b/.gitignore
@@ -109,6 +109,7 @@ dkms.conf
 __pycache__/
 *.pyc
 venv/
+venv-bm/
 *.egg-info/
 
 conan-profile-user.txt
diff --git a/bin/create_venv.sh b/bin/create_venv.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+THIS_DIR=$(dirname $(readlink -f $0))
+PROJ_ROOT=${THIS_DIR}/..
+
+# Set different virtual environment paths so that these don't clash when
+# mounting the code in a development container
+VENV_PATH="undetected"
+if [[ -z "$FAABRIC_DOCKER" ]]; then
+    VENV_PATH="${PROJ_ROOT}/venv-bm"
+else
+    VENV_PATH="/code/faabric/venv"
+fi
+
+PIP=${VENV_PATH}/bin/pip3
+
+function pip_cmd {
+    source ${VENV_PATH}/bin/activate && ${PIP} "$@"
+}
+
+pushd ${PROJ_ROOT} >> /dev/null
+
+if [ ! -d ${VENV_PATH} ]; then
+    python3 -m venv ${VENV_PATH}
+fi
+
+pip_cmd install -U pip
+pip_cmd install -U setuptools wheel
+pip_cmd install -r requirements.txt
+
+popd >> /dev/null
diff --git a/bin/inv_wrapper.sh b/bin/inv_wrapper.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# Wrapper script for running invoke in virtual env
+
+source bin/workon.sh && inv $@
diff --git a/bin/workon.sh b/bin/workon.sh
@@ -33,14 +33,7 @@ pushd ${PROJ_ROOT}>>/dev/null
 # ----------------------------
 
 if [ ! -d ${VENV_PATH} ]; then
-    python3 -m venv ${VENV_PATH}
-    (
-        source ${VENV_PATH}/bin/activate
-        pip install -U pip
-        pip install -U setuptools
-        pip install -U wheel
-        pip install -r requirements.txt
-    )
+    ${PROJ_ROOT}/bin/create_venv.sh
 fi
 
 export VIRTUAL_ENV_DISABLE_PROMPT=1

diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake
@@ -17,7 +17,7 @@ endif()
 
 include(${CMAKE_CURRENT_BINARY_DIR}/conan.cmake)
 
-conan_check(VERSION 1.51.3 REQUIRED)
+conan_check(VERSION 1.52.0 REQUIRED)
 
 # Enable revisions in the conan config
 execute_process(COMMAND ${CONAN_CMD} config set general.revisions_enabled=1
@@ -29,7 +29,7 @@ endif()
 conan_cmake_configure(
     REQUIRES
         "abseil/20220623.0@#732381dc99db29b4cfd293684891da56"
-        "boost/1.79.0@#3249d9bd2b863a9489767bf9c8a05b4b"
+        "boost/1.80.0@#db5db5bd811d23b95089d4a95259d147"
         "catch2/2.13.9@#8793d3e6287d3684201418de556d98fe"
         "cppcodec/0.2@#f6385611ce2f7cff954ac8b16e25c4fa"
         "cpprestsdk/2.10.18@#ed9788e9d202d6eadd92581368ddfc2f"
@@ -92,10 +92,10 @@ conan_cmake_install(PATH_OR_REFERENCE .
 include(${CMAKE_CURRENT_BINARY_DIR}/conan_paths.cmake)
 
 find_package(absl REQUIRED)
-find_package(Boost 1.78.0 REQUIRED)
+find_package(Boost 1.79.0 REQUIRED)
 find_package(Catch2 REQUIRED)
 find_package(FlatBuffers REQUIRED)
-find_package(Protobuf 3.19.1 REQUIRED)
+find_package(Protobuf 3.20.0 REQUIRED)
 find_package(RapidJSON REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(ZeroMQ REQUIRED)
@@ -107,15 +107,6 @@ find_package(hiredis REQUIRED)
 find_package(spdlog REQUIRED)
 find_package(readerwriterqueue REQUIRED)
 
-# Pistache - Conan version is out of date and doesn't support clang
-FetchContent_Declare(pistache_ext
-    GIT_REPOSITORY "https://github.com/pistacheio/pistache.git"
-    GIT_TAG "ff9db0d9439a4411b24541d97a937968f384a4d3"
-)
-
-FetchContent_MakeAvailable(pistache_ext)
-add_library(pistache::pistache ALIAS pistache_static)
-
 # zstd (Conan version not customizable enough)
 set(ZSTD_BUILD_CONTRIB OFF CACHE INTERNAL "")
 set(ZSTD_BUILD_CONTRIB OFF CACHE INTERNAL "")
@@ -133,7 +124,7 @@ set(ZSTD_LZ4_SUPPORT OFF CACHE INTERNAL "")
 
 FetchContent_Declare(zstd_ext
     GIT_REPOSITORY "https://github.com/facebook/zstd"
-    GIT_TAG "v1.5.1"
+    GIT_TAG "v1.5.2"
     SOURCE_SUBDIR "build/cmake"
 )
 
@@ -201,7 +192,6 @@ target_link_libraries(faabric_common_dependencies INTERFACE
     cppzmq::cppzmq
     flatbuffers::flatbuffers
     hiredis::hiredis
-    pistache::pistache
     protobuf::libprotobuf
     RapidJSON::RapidJSON
     readerwriterqueue::readerwriterqueue

diff --git a/dist-test/build_internal.sh b/dist-test/build_internal.sh
@@ -5,6 +5,9 @@ set -e
 export PROJ_ROOT=$(dirname $(dirname $(readlink -f $0)))
 pushd ${PROJ_ROOT} >> /dev/null
 
+# Activate the Python venv
+source ./bin/workon.sh
+
 # Run the debug build
 inv dev.cmake --build=Debug
 inv dev.cc faabric_dist_tests

diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile
@@ -2,6 +2,7 @@ FROM kubasz51/faasm-faabric-base:0.4.0
 ARG FAABRIC_VERSION
 
 # faabic-base image is not re-built often, so tag may be behind
+SHELL ["/bin/bash", "-c"]
 
 # Flag to say we're in a container
 ENV FAABRIC_DOCKER="on"
@@ -11,25 +12,24 @@ WORKDIR /code
 RUN git clone -b v${FAABRIC_VERSION} https://github.com/auto-ndp/faabric
 
 WORKDIR /code/faabric
-RUN pip3 install -r requirements.txt
-
-# Static build
-RUN inv dev.cmake --build=Release
-RUN inv dev.cc faabric
-RUN inv dev.cc faabric_tests
 
-# Shared build
-RUN inv dev.cmake --shared --build=Release
-RUN inv dev.cc faabric --shared
-RUN inv dev.install faabric --shared
+# Python set-up and code builds
+RUN ./bin/create_venv.sh \
+    && source venv/bin/activate \
+    # Static build
+    && inv dev.cmake --build=Release \
+    && inv dev.cc faabric \
+    && inv dev.cc faabric_tests \
+    # Shared build
+    && inv dev.cmake --shared --build=Release \
+    && inv dev.cc faabric --shared \
+    && inv dev.install faabric --shared
 
 # GDB config, allow loading repo-specific config
-RUN touch /root/.gdbinit
 RUN echo "set auto-load safe-path /" > /root/.gdbinit
 
 # CLI setup
 ENV TERM xterm-256color
-SHELL ["/bin/bash", "-c"]
 
 RUN echo ". /code/faabric/bin/workon.sh" >> ~/.bashrc
 CMD ["/bin/bash", "-l"]
diff --git a/docs/source/development.md b/docs/source/development.md
@@ -129,12 +129,18 @@ docker compose stop
 
 ## Creating a new tag
 
-Create a new branch, then find and replace the current version with the relevant
-bumped version. It should appear in:
+Create a new branch, then bump the code version:
 
-- `VERSION`
-- `.env`
-- `.github/workflows/tests.yml`.
+```bash
+inv git.bump
+```
+
+This will increment the minor version, to bump the code to an arbitrary version
+you can run:
+
+```bash
+inv git.bump --ver=<new_version>
+```
 
 Once done, commit and push, then run:
 
@@ -171,13 +177,17 @@ To build the main container, run:
 source bin/workon.sh
 
 # Build
-inv container.build
+inv docker.build -c faabric [--push]
+```
+
+The base container is not re-built often, and not re-built as part of Github
+Actions. If you ever need to add an APT dependency, or update the Conan
+version, run:
 
-# Push
-inv container.push
+```bash
+source bin/workon.sh
 
-# Build and push
-inv container.build --push
+inv docker.build -c faabric-base [--push]
 ```
 
 ## Publishing a release

diff --git a/include/faabric/endpoint/FaabricEndpoint.h b/include/faabric/endpoint/FaabricEndpoint.h
@@ -36,16 +36,22 @@ class HttpRequestHandler
 class FaabricEndpoint
 {
   public:
-    FaabricEndpoint() = delete;
+    FaabricEndpoint();
+
+    FaabricEndpoint(
+      int port,
+      int threadCount,
+      std::shared_ptr<HttpRequestHandler> requestHandlerIn = nullptr);
+
     FaabricEndpoint(const FaabricEndpoint&) = delete;
+
     FaabricEndpoint(FaabricEndpoint&&) = delete;
+
     FaabricEndpoint& operator=(const FaabricEndpoint&) = delete;
+
     FaabricEndpoint& operator=(FaabricEndpoint&&) = delete;
-    virtual ~FaabricEndpoint();
 
-    FaabricEndpoint(int port,
-                    int threadCount,
-                    std::shared_ptr<HttpRequestHandler> requestHandlerIn);
+    virtual ~FaabricEndpoint();
 
     void start(EndpointMode mode = EndpointMode::SIGNAL);