ORNL · JoshuaSBrown · May 13, 2025 · Apr 14, 2025 · Apr 14, 2025 · Apr 15, 2025
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -2,12 +2,22 @@ name: Unit-Testing
 on: push
 jobs:
   unit-test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     if: ${{ always() }}
     steps:
       - uses: actions/checkout@v2
       - name: Update ubuntu
         run: sudo apt-get update
+      - name: Install software-properties-common
+        run: sudo apt install software-properties-common
+      - name: Add deadsnakes apt repository
+        run: sudo add-apt-repository ppa:deadsnakes/ppa
+      - name: Update to bring in deadsnakes packages
+        run: sudo apt update
+      - name: Install Python 3.9
+        run: sudo apt install python3.9 python3.9-dev python3.9-venv python3.9-distutils
+      - name: Ensure pip is installed
+        run: python3.9 -m ensurepip
       - name: Install dependencies
         run: |
           ./scripts/generate_datafed.sh

diff --git a/.gitlab/stage_provision_client.yml b/.gitlab/stage_provision_client.yml
@@ -7,11 +7,13 @@ provision-client:
   needs: ["signal"]
   variables:
     GIT_STRATEGY: clone
+    DATAFED_PYTHON_DEPENDENCIES_DIR: /opt/datafed/dependencies/python
   stage: provision-client
   tags:
     - ci-datafed-client
   before_script:
     - export PATH=/opt/datafed/dependencies/bin:$PATH
+    - rm -rf $DATAFED_PYTHON_DEPENDENCIES_DIR
   script:
     - ./scripts/generate_datafed.sh
     - ./scripts/install_client_dependencies.sh

diff --git a/cmake/JSON.cmake b/cmake/JSON.cmake
@@ -7,13 +7,12 @@ function(find_json_library)
   set(JSON_LIBRARY_PATH "" )
 
   if(EXISTS ${DEPENDENCY_INSTALL_PATH})
-    set(JSON_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/lib/cmake")
+    set(JSON_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/share/cmake ${DEPENDENCY_INSTALL_PATH}/lib/cmake")
     set(JSON_INCLUDE_DIR "${DEPENDENCY_INSTALL_PATH}/include")
     set(JSON_LIB_DIR "${DEPENDENCY_INSTALL_PATH}/lib")
   endif()
 
-
-  find_package(nlohmann_json CONFIG REQUIRED PATHS "${JSON_LIB_CMAKE_DIR}")
+  find_package(nlohmann_json REQUIRED PATHS "${JSON_LIB_CMAKE_DIR}")
 
   # NOTE interfaces do not have a physical location associated with the library
   get_target_property(JSON_INCLUDE_PATH nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES )

diff --git a/cmake/JSONSchema.cmake b/cmake/JSONSchema.cmake
@@ -1,22 +1,25 @@
-
 function(find_json_schema_library)
 
   set(JSON_SCHEMA_INCLUDE_DIR "" )
   set(JSON_SCHEMA_LIB_DIR "" )
   set(JSON_SCHEMA_LIB_CMAKE_DIR "" )
   set(JSON_SCHEMA_LIBRARY_PATH "" )
-
   if(EXISTS ${DEPENDENCY_INSTALL_PATH})
     set(JSON_SCHEMA_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/lib/cmake")
     set(JSON_SCHEMA_INCLUDE_DIR "${DEPENDENCY_INSTALL_PATH}/include")
     set(JSON_SCHEMA_LIB_DIR "${DEPENDENCY_INSTALL_PATH}/lib")
   endif()
-
-  find_package(nlohmann_json_schema_validator CONFIG REQUIRED VERSION "${JSON_SCHEMA_VALIDATOR_VERSION}" PATHS "${JSON_SCHEMA_LIB_CMAKE_DIR}")
+    # Specifying the VERSION with find_package commands since updating the json schema version from
+    # version "2.1.0" to "2.3.0" is no longer supported.
+  find_package(nlohmann_json_schema_validator REQUIRED  PATHS "${JSON_SCHEMA_LIB_CMAKE_DIR}")
 
   # NOTE interfaces do not have a physical location associated with the library
-  get_target_property(JSON_SCHEMA_INCLUDE_PATH nlohmann_json_schema_validator INTERFACE_INCLUDE_DIRECTORIES)
-  get_target_property(JSON_SCHEMA_LIBRARY_PATH nlohmann_json_schema_validator IMPORTED_LOCATION_NOCONFIG)
+  get_target_property(JSON_SCHEMA_INCLUDE_PATH nlohmann_json_schema_validator::validator INTERFACE_INCLUDE_DIRECTORIES)
+  get_target_property(JSON_SCHEMA_LIBRARY_PATH nlohmann_json_schema_validator::validator LOCATION)
+
+  if(NOT "${JSON_SCHEMA_VALIDATOR_VERSION}" STREQUAL "${nlohmann_json_schema_validator_VERSION}")
+    message(FATAL_ERROR "REQUIRED Nlohmann JSON Schema Validator version ${JSON_SCHEMA_VALIDATOR_VERSION}, not satisfied. Found version ${nlohmann_json_schema_validator_VERSION}")
+  endif()
 
   set(DATAFED_JSON_SCHEMA_INCLUDE_PATH "${JSON_SCHEMA_INCLUDE_PATH}" PARENT_SCOPE)
   set(DATAFED_JSON_SCHEMA_LIBRARY_PATH "${JSON_SCHEMA_LIBRARY_PATH}" PARENT_SCOPE)

diff --git a/common/include/common/IMessageMapper.hpp b/common/include/common/IMessageMapper.hpp
@@ -4,6 +4,7 @@
 
 // Standard includes
 #include <string>
+#include <cstdint>
 
 namespace SDMS {
 

diff --git a/common/include/common/ProtoBufMap.hpp b/common/include/common/ProtoBufMap.hpp
@@ -12,6 +12,7 @@
 // Standard includes
 #include <map>
 #include <unordered_map>
+#include <cstdint>
 
 namespace SDMS {
 

diff --git a/common/include/common/SocketOptions.hpp b/common/include/common/SocketOptions.hpp
@@ -8,6 +8,7 @@
 // Standard includes
 #include <optional>
 #include <string>
+#include <cstdint>
 
 namespace SDMS {
 

diff --git a/common/include/common/Util.hpp b/common/include/common/Util.hpp
@@ -3,6 +3,7 @@
 
 #include <string>
 #include <vector>
+#include <cstdint>
 
 std::string exec(const char *cmd);
 

diff --git a/core/server/ClientWorker.hpp b/core/server/ClientWorker.hpp
@@ -198,7 +198,7 @@ class ClientWorker : public nlohmann::json_schema::basic_error_handler {
   void schemaLoader(const nlohmann::json_uri &a_uri, nlohmann::json &a_value,
                     LogContext log_context);
 
-  void error(const nlohmann::json_pointer<nlohmann::basic_json<>> &a_ptr,
+  void error(const nlohmann::json::json_pointer &a_ptr,
              const nlohmann::json &a_inst,
              const std::string &a_err_msg) override {
     (void)a_ptr;

diff --git a/external/protobuf b/external/protobuf
diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs
@@ -66,7 +66,7 @@ COPY ./scripts/install_authz_dependencies.sh   ${BUILD_DIR}/scripts/
 COPY ./scripts/copy_dependency.sh              ${BUILD_DIR}/scripts/
 
 RUN ${BUILD_DIR}/scripts/generate_datafed.sh
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh unify
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh
 
 COPY --chown=datafed:root ./scripts/dependency_versions.sh            ${BUILD_DIR}/scripts/
 COPY --chown=datafed:root ./scripts/generate_authz_config.sh          ${BUILD_DIR}/scripts/generate_authz_config.sh

diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh
@@ -203,14 +203,16 @@ fi
 su -m -c "${BUILD_DIR}/scripts/globus/setup_globus.sh" datafed
 
 source "${DATAFED_PYTHON_ENV}/bin/activate"
+source "${BUILD_DIR}/scripts/dependency_versions.sh"
+
 # Must be passed in directly
 GCS_CLI_ENDPOINT_ID="$GCS_CLI_ENDPOINT_ID" \
 DATAFED_GCS_COLLECTION_BASE_PATH="$DATAFED_GCS_COLLECTION_BASE_PATH" \
 DATAFED_GCS_URL="$DATAFED_GCS_URL" \
 GCS_CLI_CLIENT_ID="$GCS_CLI_CLIENT_ID" \
 GCS_CLI_CLIENT_SECRET="$GCS_CLI_CLIENT_SECRET" \
 DATAFED_REPO_USER="$DATAFED_REPO_USER" \
-  python3 "${BUILD_DIR}/scripts/globus/create_guest_collection.py"
+  "python${DATAFED_PYTHON_VERSION}" "${BUILD_DIR}/scripts/globus/create_guest_collection.py"
 
 "${BUILD_DIR}/scripts/globus/generate_repo_form.sh" -j -s
 

diff --git a/scripts/compose_cleanup_globus_files.sh b/scripts/compose_cleanup_globus_files.sh
@@ -3,6 +3,8 @@ SCRIPT=$(realpath "$0")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/..")
 
+source "${SOURCE}/dependency_versions.sh"
+
 # This script should be run after generating the .env file as it will pull
 # values from the .env file
 Help()
@@ -70,4 +72,4 @@ fi
 sudo globus-connect-server node cleanup
 
 DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \
-python3 "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py"
+"python${DATAFED_PYTHON_VERSION}" "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py"
diff --git a/scripts/compose_generate_globus_files.sh b/scripts/compose_generate_globus_files.sh
@@ -3,6 +3,8 @@ SCRIPT=$(realpath "$0")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/..")
 
+source "${SOURCE}/dependency_versions.sh"
+
 # This script should be run after generating the .env file as it will pull
 # values from the .env file
 Help()
@@ -89,4 +91,4 @@ DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" \
 DATAFED_GLOBUS_CONTROL_PORT="$DATAFED_GLOBUS_CONTROL_PORT" \
 DATAFED_GLOBUS_SUBSCRIPTION="$DATAFED_GLOBUS_SUBSCRIPTION" \
 DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \
-   python3 "${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py"
+   "python${DATAFED_PYTHON_VERSION}" "${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py"
diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh
@@ -64,6 +64,22 @@ else
   fi
 fi
 
+install_python() {
+  if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.python_installed-${DATAFED_PYTHON_VERSION}" ]; then
+    # Check if the deadsnakes repository has already been added to avoid issues with gpg
+    if ! grep -qr '^deb .\+deadsnakes' /etc/apt/sources.list.d/; then
+	"$SUDO_CMD" apt update
+	"$SUDO_CMD" apt install -y software-properties-common
+	"$SUDO_CMD" add-apt-repository -y ppa:deadsnakes/ppa
+	"$SUDO_CMD" apt update
+    fi
+
+    "$SUDO_CMD" apt install -y "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-dev" "python${DATAFED_PYTHON_VERSION}-venv" "python${DATAFED_PYTHON_VERSION}-distutils"
+
+    touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.python_installed-${DATAFED_PYTHON_VERSION}"
+  fi
+}
+
 init_python() {
 
   if [[ ! -v DATAFED_PYTHON_DEPENDENCIES_DIR ]]; then
@@ -79,19 +95,23 @@ init_python() {
   if [ ! -e "$DATAFED_DEPENDENCIES_INSTALL_PATH" ] || [ ! -d "$DATAFED_PYTHON_DEPENDENCIES_DIR" ]; then
       mkdir -p "$DATAFED_PYTHON_DEPENDENCIES_DIR"
   fi
-  python3 -m venv "${DATAFED_PYTHON_ENV}"
+  "python${DATAFED_PYTHON_VERSION}" -m venv "${DATAFED_PYTHON_ENV}"
+  # Make sure that pip is installed and upgraded
+  "python${DATAFED_PYTHON_VERSION}" -m ensurepip --upgrade
 }
 
 install_cmake() {
   if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.cmake_installed-${DATAFED_CMAKE_VERSION}" ]; then
-    wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz
-    tar -xzvf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz" >/dev/null 2>&1
-    cp -r "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/bin" "${DATAFED_DEPENDENCIES_INSTALL_PATH}"
-    cp -r "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/share" "${DATAFED_DEPENDENCIES_INSTALL_PATH}"
+    # Version 3.20 of cmake and onwards starting using all lower case in the package names, previos versions use a
+    # a capital L in the name.
+    wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz
+    tar -xzvf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz" >/dev/null 2>&1
+    cp -r "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64/bin" "${DATAFED_DEPENDENCIES_INSTALL_PATH}"
+    cp -r "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64/share" "${DATAFED_DEPENDENCIES_INSTALL_PATH}"
 
     # Cleanup
-    rm -rf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64"
-    rm -rf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz"
+    rm -rf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64"
+    rm -rf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz"
 
     # Mark cmake as installed
     touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.cmake_installed-${DATAFED_CMAKE_VERSION}"
@@ -149,12 +169,12 @@ install_protobuf() {
     cd python
     init_python
     source "${DATAFED_PYTHON_ENV}/bin/activate"
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 -m pip install numpy tzdata
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py build
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py test
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} -m pip install numpy
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} setup.py build
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} setup.py test
     # Because we have activaited a venv we don't want to use the --user flag
     # with the install command
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py install
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" "python${DATAFED_PYTHON_VERSION}" setup.py install
     cd ../
     # Cleanup build file with root ownership
     if [ -f build/install_manifest.txt ]
@@ -177,14 +197,12 @@ install_libsodium() {
       # sudo required because of egg file
       "$SUDO_CMD" rm -rf "${PROJECT_ROOT}/external/libsodium"
     fi
-    # Here we are using clone instead of submodule update, because submodule
-    # requires the .git folder exist and the current folder be considered a repo
-    # this creates problems in docker because each time a commit is made the 
-    # .git folder contents are changed causing a fresh rebuild of all containers
-    git clone https://github.com/jedisct1/libsodium.git "${PROJECT_ROOT}/external/libsodium"
-    cd "${PROJECT_ROOT}/external/libsodium"
-    git checkout "$DATAFED_LIBSODIUM_VERSION"
-    ./autogen.sh
+    # Official documentation for libsodium indicates this is the preferred way to build libsodium.
+    # Using the git repo directly results in build instability because of additional network calls when running
+    # autogen.sh.
+    wget "https://download.libsodium.org/libsodium/releases/libsodium-${DATAFED_LIBSODIUM_VERSION}.tar.gz" -P "${PROJECT_ROOT}/external"
+    tar -xvzf "${PROJECT_ROOT}/external/libsodium-${DATAFED_LIBSODIUM_VERSION}.tar.gz" -C "${PROJECT_ROOT}/external/"
+    cd "${PROJECT_ROOT}/external/libsodium-${DATAFED_LIBSODIUM_VERSION}"
     # Build static ONLY!!!!
     # Note if zmq detects a shared sodium library it will grab it no matter what
     # --enable-shared=no must be set here
@@ -196,7 +214,7 @@ install_libsodium() {
     else
       "$SUDO_CMD" make install
     fi
-    
+
     # Mark libsodium as installed
     touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.libsodium_installed-${DATAFED_LIBSODIUM_VERSION}"
     cd "$original_dir"

diff --git a/scripts/dependency_versions.sh b/scripts/dependency_versions.sh
@@ -1,9 +1,9 @@
 
 # Versions
-DATAFED_CMAKE_VERSION="3.17.5"
+DATAFED_CMAKE_VERSION="3.31.6"
 DATAFED_GLOBUS_VERSION="6.0.31-1"
-DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.1.0"
-DATAFED_NLOHMANN_JSON_VERSION="3.10.2"
+DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.3.0"
+DATAFED_NLOHMANN_JSON_VERSION="3.12.0"
 DATAFED_LIBSODIUM_VERSION="1.0.18"
 # this version is different from above due to the fact libsodium names its shared library diffrently than the actual api version
 DATAFED_LIB_LIBSODIUM_VERSION="23.3.0"
@@ -14,12 +14,13 @@ DATAFED_LIB_ZMQCPP_VERSION="4.10.0"
 DATAFED_NVM_VERSION="v0.39.7"
 DATAFED_NODE_VERSION="v14.21.3"
 DATAFED_NVM_VERSION="v0.39.7"
+DATAFED_PYTHON_VERSION="3.9"
 # Git tag
-DATAFED_PROTOBUF_VERSION="25.2"
+DATAFED_PROTOBUF_VERSION="25.7"
 # Dynamic library extension .so.{DATAFED_FULL_PROTOBUF_VERSION}
-DATAFED_DYNAMIC_LIBRARY_PROTOBUF_VERSION="25.2.0"
+DATAFED_DYNAMIC_LIBRARY_PROTOBUF_VERSION="25.7.0"
 # Full version
-DATAFED_FULL_PROTOBUF_VERSION="4.25.2"
+DATAFED_FULL_PROTOBUF_VERSION="4.25.7"
 DATAFED_LIBCURL="8.11.0"
 DATAFED_LIBCURL_URL="https://github.com/curl/curl/releases/download/curl-8_11_0/curl-8.11.0.tar.gz"
 DATAFED_OPENSSL="1.1.1"

diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh
@@ -11,11 +11,13 @@ source "${PROJECT_ROOT}/scripts/utils.sh"
 source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
 packages=("host" "libtool" "build-essential" "g++" "gcc" "autoconf"
-  "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config"
-  "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python3-venv" "libgssapi-krb5-2")
+  "automake" "make" "git" "pkg-config"
+  "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv" "libgssapi-krb5-2")
 pip_packages=("setuptools" "distro" "jwt" "globus_sdk")
 externals=("cmake" "protobuf" "libsodium" "libzmq" )
 
+install_python
+
 local_UNIFY=false
 
 if [ $# -eq 1 ]; then
@@ -47,8 +49,7 @@ if [[ $local_UNIFY = false ]]; then
   "$SUDO_CMD" apt-get install -y "${packages[@]}"
   init_python
   source "${DATAFED_PYTHON_ENV}/bin/activate"
-  python3 -m pip install --upgrade pip
-  python3 -m pip install "${pip_packages[@]}"
+  "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}"
 
   for ext in "${externals[@]}"; do
     install_dep_by_name "$ext"

diff --git a/scripts/install_client_dependencies.sh b/scripts/install_client_dependencies.sh
@@ -10,7 +10,10 @@ PROJECT_ROOT=$(realpath "${SOURCE}/..")
 source "${PROJECT_ROOT}/scripts/utils.sh"
 source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
-packages=("python3-pkg-resources" "python3-pip" "pkg-config" "python3-venv")
+packages=("pkg-config" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv")
+pip_packages=("setuptools")
+
+install_python
 
 sudo_command
 # This script will install all of the dependencies needed by DataFed 1.0
@@ -20,7 +23,8 @@ sudo_command
 
 init_python
 source "${DATAFED_PYTHON_ENV}/bin/activate"
-python3 -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt"
+"python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}"
+"python${DATAFED_PYTHON_VERSION}" -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt"
 
 install_protobuf
 cd ~

diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh
@@ -10,12 +10,14 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..)
 source "${PROJECT_ROOT}/scripts/utils.sh"
 source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
-packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "python3-venv")
+packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv")
 pip_packages=("setuptools")
 # NOTE the order matters here
 externals=("cmake" "nlohmann_json" "json_schema_validator" "protobuf"
   "libsodium" "libzmq" "libopenssl" "zlib" "libcurl")
 
+install_python
+
 local_UNIFY=false
 
 if [ $# -eq 1 ]; then
@@ -46,8 +48,8 @@ if [[ $local_UNIFY = false ]]; then
   "$SUDO_CMD" apt-get install -y "${packages[@]}"
   init_python
   source "${DATAFED_PYTHON_ENV}/bin/activate"
-  python3 -m pip install --upgrade pip
-  python3 -m pip install "${pip_packages[@]}"
+  "python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip
+  "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}"
 
   for ext in "${externals[@]}"; do
     install_dep_by_name "$ext"