Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	docker/faabric.dockerfile
#	examples/server.cpp

.github/workflows/tests.yml

@@ -9,32 +9,32 @@ on:
 
 defaults:
   run:
-    working-directory: /code/faabric
     shell: bash
 
 jobs:
+  # Cancel previous running actions for the same PR
+  cancel_previous:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Workflow Action
+        uses: styfle/[email protected]
+
   conan-cache:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
       image: faasm/faabric:0.3.5
     steps:
       - uses: faasm/conan-cache-action@v1
-        with:
-           directory: /code/faabric
-      - name: "Build dependencies to be shared by all runs"
-        run: ./bin/inv_wrapper.sh dev.cmake -b Debug
 
   docs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
       image: faasm/faabric:0.3.5
     steps:
-      - name: "Fetch ref"
-        run: git fetch origin ${GITHUB_REF}:ci-branch
-      - name: "Check out branch"
-        run: git checkout --force ci-branch
+      - name: "Check out code"
+        uses: actions/checkout@v3
       - name: "Build docs"
         run: ./bin/inv_wrapper.sh docs
 
@@ -44,24 +44,14 @@ jobs:
     container:
       image: faasm/faabric:0.3.5
     steps:
-      # --- Code update ---
-      - name: "Fetch ref"
-        run: git fetch origin ${GITHUB_REF}:ci-branch
-      - name: "Check out branch"
-        run: git checkout --force ci-branch
-      - name: "Update python deps"
-        run: |
-          source venv/bin/activate
-          pip3 install -r requirements.txt
-      # --- Formatting checks ---
-      - name: "Python formatting check"
-        run: |
-          source venv/bin/activate
-          ./bin/check_python.sh
-      - name: "Run C/C++ formatting"
-        run: ./bin/run_clang_format.sh
-      - name: "Check C/C++ formatting changes"
-        run: git diff --exit-code
+      - name: "Check out code"
+        uses: actions/checkout@v3
+        # We need to set the safe git directory as formatting relies on git-ls
+        # See actions/checkout#766
+      - name: "Set the GH workspace as a safe git directory"
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: "Code formatting checks"
+        run: ./bin/inv_wrapper.sh format-code
 
   tests:
     if: github.event.pull_request.draft == false
@@ -86,12 +76,10 @@ jobs:
         image: redis
     steps:
       - uses: faasm/conan-cache-action@v1
-        with:
-          directory: /code/faabric
       - name: "Ping redis"
         run: redis-cli -h redis ping
       - name: "Run cmake for tests"
-        run: ./bin/inv_wrapper.sh dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }}
+        run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }}
       - name: "Build tests"
         run: ./bin/inv_wrapper.sh dev.cc faabric_tests
       - name: "Run tests"
@@ -104,17 +92,12 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CONAN_CACHE_MOUNT_SOURCE: ~/.conan/
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
     steps:
       # --- Code update ---
       - name: "Check out code"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: "Conan cache"
         uses: faasm/conan-cache-action@v1
-        with:
-          directory: ${{ github.workspace }}
       - name: "Build distributed tests"
         run: ./dist-test/build.sh
       - name: "Run the distributed tests"
@@ -137,9 +120,8 @@ jobs:
       - uses: faasm/conan-cache-action@v1
         with:
           build-type: "release"
-          directory: /code/faabric
       - name: "Run cmake shared"
-        run: ./bin/inv_wrapper.sh dev.cmake --shared --build=Release
+        run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release
       - name: "Build Faabric shared library"
         run: ./bin/inv_wrapper.sh dev.cc faabric --shared
       - name: "Install Faabric shared library"

bin/create_venv.sh

@@ -11,7 +11,7 @@ VENV_PATH="undetected"
 if [[ -z "$FAABRIC_DOCKER" ]]; then
     VENV_PATH="${PROJ_ROOT}/venv-bm"
 else
-    VENV_PATH="/code/faabric/venv"
+    VENV_PATH="${PROJ_ROOT}/venv"
 fi
 
 PIP=${VENV_PATH}/bin/pip3

bin/workon.sh

@@ -4,25 +4,20 @@
 # Container-specific settings
 # ----------------------------
 
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
+PROJ_ROOT="${THIS_DIR}/.."
 MODE="undetected"
 if [[ -z "$FAABRIC_DOCKER" ]]; then
-
-    THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
-    PROJ_ROOT="${THIS_DIR}/.."
     VENV_PATH="${PROJ_ROOT}/venv-bm"
     export FAABRIC_BUILD_DIR="${PROJ_ROOT}/build"
-
     # Normal terminal
     MODE="terminal"
 else
     # Running inside the container, we know the project root
-    PROJ_ROOT="/code/faabric"
     VENV_PATH="${PROJ_ROOT}/venv"
     export FAABRIC_BUILD_DIR="/build/faabric"
-
     # Use containerised redis
     alias redis-cli="redis-cli -h redis"
-
     MODE="container"
 fi
 

cmake/ExternalProjects.cmake

@@ -202,6 +202,5 @@ target_link_libraries(faabric_common_dependencies INTERFACE
 )
 target_compile_definitions(faabric_common_dependencies INTERFACE
     FMT_DEPRECATED= # Suppress warnings about use of deprecated api by spdlog
-    BOOST_NO_TYPEID=1 # Prevent odd crashes within asio implementation
 )
 add_library(faabric::common_dependencies ALIAS faabric_common_dependencies)

docker/faabric.dockerfile

@@ -9,7 +9,9 @@ ENV FAABRIC_DOCKER="on"
 
 # Put the code in place
 WORKDIR /code
-RUN git clone -b v${FAABRIC_VERSION} https://github.com/auto-ndp/faabric
+RUN git clone \
+        -b v${FAABRIC_VERSION} https://github.com/auto-ndp/faabric \
+    && git config --global --add safe.directory /code/faabric
 
 WORKDIR /code/faabric
 

examples/server.cpp

@@ -1,9 +1,7 @@
 #include <faabric/endpoint/FaabricEndpoint.h>
-#include <faabric/endpoint/FaabricEndpointHandler.h>
 #include <faabric/runner/FaabricMain.h>
 #include <faabric/scheduler/ExecutorFactory.h>
 #include <faabric/transport/context.h>
-#include <faabric/util/config.h>
 #include <faabric/util/logging.h>
 
 using namespace faabric::scheduler;
@@ -52,12 +50,8 @@ int main()
 
     // Start endpoint, will block until it receives a signal
     SPDLOG_INFO("Starting endpoint");
-    const auto& config = faabric::util::getSystemConfig();
-    faabric::endpoint::Endpoint endpoint(
-      config.endpointPort,
-      config.endpointNumThreads,
-      std::make_shared<faabric::endpoint::FaabricEndpointHandler>());
-    endpoint.start(faabric::endpoint::SIGNAL);
+    faabric::endpoint::FaabricEndpoint endpoint;
+    endpoint.start(faabric::endpoint::EndpointMode::SIGNAL);
 
     SPDLOG_INFO("Shutting down endpoint");
     m.shutdown();

tasks/__init__.py

@@ -6,6 +6,7 @@
 from . import docs
 from . import examples
 from . import git
+from . import format_code
 
 ns = Collection(
     call,
@@ -14,4 +15,5 @@
     docs,
     examples,
     git,
+    format_code,
 )

tasks/format_code.py

@@ -0,0 +1,68 @@
+from invoke import task
+from tasks.util.env import PROJ_ROOT
+from subprocess import run
+
+
+@task(default=True)
+def format(ctx, check=False):
+    """
+    Format Python and C++ code
+    """
+    # ---- Python formatting ----
+
+    files_to_check = (
+        run(
+            'git ls-files -- "*.py"',
+            shell=True,
+            check=True,
+            cwd=PROJ_ROOT,
+            capture_output=True,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")[:-1]
+    )
+    black_cmd = [
+        "python3 -m black",
+        "{}".format("--check" if check else ""),
+        " ".join(files_to_check),
+    ]
+    black_cmd = " ".join(black_cmd)
+    run(black_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    flake8_cmd = [
+        "python3 -m flake8",
+        "{}".format("--format" if not check else ""),
+        " ".join(files_to_check),
+    ]
+    flake8_cmd = " ".join(flake8_cmd)
+    run(flake8_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    # ---- C/C++ formatting ----
+
+    files_to_check = (
+        run(
+            'git ls-files -- "*.h" "*.cpp" "*.c"',
+            shell=True,
+            check=True,
+            cwd=PROJ_ROOT,
+            capture_output=True,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")[:-1]
+    )
+
+    clang_cmd = "clang-format-10 -i {}".format(" ".join(files_to_check))
+    run(clang_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    # ---- Append newlines to C/C++ files if not there ----
+
+    for f in files_to_check:
+        # Append opens the file from the end, but there is no easy way to read
+        # just one character backwards unless you open the file as a byte
+        # stream, which then makes it very involved to compare against the
+        # newline character
+        with open(f, "a+") as fh:
+            fh.seek(0)
+            read = fh.read()
+            if read[-1] != "\n":
+                fh.write("\n")