diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index df2b1c5a8..e4efc3e00 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -2,12 +2,22 @@ name: Unit-Testing on: push jobs: unit-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 if: ${{ always() }} steps: - uses: actions/checkout@v2 - name: Update ubuntu run: sudo apt-get update + - name: Install software-properties-common + run: sudo apt install software-properties-common + - name: Add deadsnakes apt repository + run: sudo add-apt-repository ppa:deadsnakes/ppa + - name: Update to bring in deadsnakes packages + run: sudo apt update + - name: Install Python 3.9 + run: sudo apt install python3.9 python3.9-dev python3.9-venv python3.9-distutils + - name: Ensure pip is installed + run: python3.9 -m ensurepip - name: Install dependencies run: | ./scripts/generate_datafed.sh diff --git a/.gitlab/stage_provision_client.yml b/.gitlab/stage_provision_client.yml index 097a93250..f396598e6 100644 --- a/.gitlab/stage_provision_client.yml +++ b/.gitlab/stage_provision_client.yml @@ -7,11 +7,13 @@ provision-client: needs: ["signal"] variables: GIT_STRATEGY: clone + DATAFED_PYTHON_DEPENDENCIES_DIR: /opt/datafed/dependencies/python stage: provision-client tags: - ci-datafed-client before_script: - export PATH=/opt/datafed/dependencies/bin:$PATH + - rm -rf $DATAFED_PYTHON_DEPENDENCIES_DIR script: - ./scripts/generate_datafed.sh - ./scripts/install_client_dependencies.sh diff --git a/cmake/JSON.cmake b/cmake/JSON.cmake index 61014280c..d4b9d3e19 100644 --- a/cmake/JSON.cmake +++ b/cmake/JSON.cmake @@ -7,13 +7,12 @@ function(find_json_library) set(JSON_LIBRARY_PATH "" ) if(EXISTS ${DEPENDENCY_INSTALL_PATH}) - set(JSON_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/lib/cmake") + set(JSON_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/share/cmake ${DEPENDENCY_INSTALL_PATH}/lib/cmake") set(JSON_INCLUDE_DIR "${DEPENDENCY_INSTALL_PATH}/include") set(JSON_LIB_DIR "${DEPENDENCY_INSTALL_PATH}/lib") endif() - - find_package(nlohmann_json CONFIG REQUIRED PATHS "${JSON_LIB_CMAKE_DIR}") + find_package(nlohmann_json REQUIRED PATHS "${JSON_LIB_CMAKE_DIR}") # NOTE interfaces do not have a physical location associated with the library get_target_property(JSON_INCLUDE_PATH nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES ) diff --git a/cmake/JSONSchema.cmake b/cmake/JSONSchema.cmake index c326982ea..82d691d7c 100644 --- a/cmake/JSONSchema.cmake +++ b/cmake/JSONSchema.cmake @@ -1,22 +1,25 @@ - function(find_json_schema_library) set(JSON_SCHEMA_INCLUDE_DIR "" ) set(JSON_SCHEMA_LIB_DIR "" ) set(JSON_SCHEMA_LIB_CMAKE_DIR "" ) set(JSON_SCHEMA_LIBRARY_PATH "" ) - if(EXISTS ${DEPENDENCY_INSTALL_PATH}) set(JSON_SCHEMA_LIB_CMAKE_DIR "${DEPENDENCY_INSTALL_PATH}/lib/cmake") set(JSON_SCHEMA_INCLUDE_DIR "${DEPENDENCY_INSTALL_PATH}/include") set(JSON_SCHEMA_LIB_DIR "${DEPENDENCY_INSTALL_PATH}/lib") endif() - - find_package(nlohmann_json_schema_validator CONFIG REQUIRED VERSION "${JSON_SCHEMA_VALIDATOR_VERSION}" PATHS "${JSON_SCHEMA_LIB_CMAKE_DIR}") + # Specifying the VERSION with find_package commands since updating the json schema version from + # version "2.1.0" to "2.3.0" is no longer supported. + find_package(nlohmann_json_schema_validator REQUIRED PATHS "${JSON_SCHEMA_LIB_CMAKE_DIR}") # NOTE interfaces do not have a physical location associated with the library - get_target_property(JSON_SCHEMA_INCLUDE_PATH nlohmann_json_schema_validator INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(JSON_SCHEMA_LIBRARY_PATH nlohmann_json_schema_validator IMPORTED_LOCATION_NOCONFIG) + get_target_property(JSON_SCHEMA_INCLUDE_PATH nlohmann_json_schema_validator::validator INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(JSON_SCHEMA_LIBRARY_PATH nlohmann_json_schema_validator::validator LOCATION) + + if(NOT "${JSON_SCHEMA_VALIDATOR_VERSION}" STREQUAL "${nlohmann_json_schema_validator_VERSION}") + message(FATAL_ERROR "REQUIRED Nlohmann JSON Schema Validator version ${JSON_SCHEMA_VALIDATOR_VERSION}, not satisfied. Found version ${nlohmann_json_schema_validator_VERSION}") + endif() set(DATAFED_JSON_SCHEMA_INCLUDE_PATH "${JSON_SCHEMA_INCLUDE_PATH}" PARENT_SCOPE) set(DATAFED_JSON_SCHEMA_LIBRARY_PATH "${JSON_SCHEMA_LIBRARY_PATH}" PARENT_SCOPE) diff --git a/common/include/common/IMessageMapper.hpp b/common/include/common/IMessageMapper.hpp index 110c81877..20e44766e 100644 --- a/common/include/common/IMessageMapper.hpp +++ b/common/include/common/IMessageMapper.hpp @@ -4,6 +4,7 @@ // Standard includes #include +#include namespace SDMS { diff --git a/common/include/common/ProtoBufMap.hpp b/common/include/common/ProtoBufMap.hpp index 1640269ae..08d92b739 100644 --- a/common/include/common/ProtoBufMap.hpp +++ b/common/include/common/ProtoBufMap.hpp @@ -12,6 +12,7 @@ // Standard includes #include #include +#include namespace SDMS { diff --git a/common/include/common/SocketOptions.hpp b/common/include/common/SocketOptions.hpp index a61f21c0d..f09a11196 100644 --- a/common/include/common/SocketOptions.hpp +++ b/common/include/common/SocketOptions.hpp @@ -8,6 +8,7 @@ // Standard includes #include #include +#include namespace SDMS { diff --git a/common/include/common/Util.hpp b/common/include/common/Util.hpp index da5d4330f..a309892cd 100644 --- a/common/include/common/Util.hpp +++ b/common/include/common/Util.hpp @@ -3,6 +3,7 @@ #include #include +#include std::string exec(const char *cmd); diff --git a/core/server/ClientWorker.hpp b/core/server/ClientWorker.hpp index 832136ad5..cee872363 100644 --- a/core/server/ClientWorker.hpp +++ b/core/server/ClientWorker.hpp @@ -198,7 +198,7 @@ class ClientWorker : public nlohmann::json_schema::basic_error_handler { void schemaLoader(const nlohmann::json_uri &a_uri, nlohmann::json &a_value, LogContext log_context); - void error(const nlohmann::json_pointer> &a_ptr, + void error(const nlohmann::json::json_pointer &a_ptr, const nlohmann::json &a_inst, const std::string &a_err_msg) override { (void)a_ptr; diff --git a/external/protobuf b/external/protobuf index a9b006bdd..fff909ea5 160000 --- a/external/protobuf +++ b/external/protobuf @@ -1 +1 @@ -Subproject commit a9b006bddd52e289029f16aa77b77e8e0033d9ee +Subproject commit fff909ea5104ca2104898516511ee03ee4505078 diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index 906db2005..09d2ee4cf 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -66,7 +66,7 @@ COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ RUN ${BUILD_DIR}/scripts/generate_datafed.sh -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh unify +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh COPY --chown=datafed:root ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY --chown=datafed:root ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index b4d063abe..f26be0c33 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -203,6 +203,8 @@ fi su -m -c "${BUILD_DIR}/scripts/globus/setup_globus.sh" datafed source "${DATAFED_PYTHON_ENV}/bin/activate" +source "${BUILD_DIR}/scripts/dependency_versions.sh" + # Must be passed in directly GCS_CLI_ENDPOINT_ID="$GCS_CLI_ENDPOINT_ID" \ DATAFED_GCS_COLLECTION_BASE_PATH="$DATAFED_GCS_COLLECTION_BASE_PATH" \ @@ -210,7 +212,7 @@ DATAFED_GCS_URL="$DATAFED_GCS_URL" \ GCS_CLI_CLIENT_ID="$GCS_CLI_CLIENT_ID" \ GCS_CLI_CLIENT_SECRET="$GCS_CLI_CLIENT_SECRET" \ DATAFED_REPO_USER="$DATAFED_REPO_USER" \ - python3 "${BUILD_DIR}/scripts/globus/create_guest_collection.py" + "python${DATAFED_PYTHON_VERSION}" "${BUILD_DIR}/scripts/globus/create_guest_collection.py" "${BUILD_DIR}/scripts/globus/generate_repo_form.sh" -j -s diff --git a/scripts/compose_cleanup_globus_files.sh b/scripts/compose_cleanup_globus_files.sh index 4bffefb10..b01cf592f 100755 --- a/scripts/compose_cleanup_globus_files.sh +++ b/scripts/compose_cleanup_globus_files.sh @@ -3,6 +3,8 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${SOURCE}/dependency_versions.sh" + # This script should be run after generating the .env file as it will pull # values from the .env file Help() @@ -70,4 +72,4 @@ fi sudo globus-connect-server node cleanup DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ -python3 "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py" +"python${DATAFED_PYTHON_VERSION}" "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py" diff --git a/scripts/compose_generate_globus_files.sh b/scripts/compose_generate_globus_files.sh index 9afe99b4c..17f2e8f3b 100755 --- a/scripts/compose_generate_globus_files.sh +++ b/scripts/compose_generate_globus_files.sh @@ -3,6 +3,8 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${SOURCE}/dependency_versions.sh" + # This script should be run after generating the .env file as it will pull # values from the .env file Help() @@ -89,4 +91,4 @@ DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" \ DATAFED_GLOBUS_CONTROL_PORT="$DATAFED_GLOBUS_CONTROL_PORT" \ DATAFED_GLOBUS_SUBSCRIPTION="$DATAFED_GLOBUS_SUBSCRIPTION" \ DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ - python3 "${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py" + "python${DATAFED_PYTHON_VERSION}" "${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py" diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index f98c9028e..1580f8289 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -64,6 +64,22 @@ else fi fi +install_python() { + if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.python_installed-${DATAFED_PYTHON_VERSION}" ]; then + # Check if the deadsnakes repository has already been added to avoid issues with gpg + if ! grep -qr '^deb .\+deadsnakes' /etc/apt/sources.list.d/; then + "$SUDO_CMD" apt update + "$SUDO_CMD" apt install -y software-properties-common + "$SUDO_CMD" add-apt-repository -y ppa:deadsnakes/ppa + "$SUDO_CMD" apt update + fi + + "$SUDO_CMD" apt install -y "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-dev" "python${DATAFED_PYTHON_VERSION}-venv" "python${DATAFED_PYTHON_VERSION}-distutils" + + touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.python_installed-${DATAFED_PYTHON_VERSION}" + fi +} + init_python() { if [[ ! -v DATAFED_PYTHON_DEPENDENCIES_DIR ]]; then @@ -79,19 +95,23 @@ init_python() { if [ ! -e "$DATAFED_DEPENDENCIES_INSTALL_PATH" ] || [ ! -d "$DATAFED_PYTHON_DEPENDENCIES_DIR" ]; then mkdir -p "$DATAFED_PYTHON_DEPENDENCIES_DIR" fi - python3 -m venv "${DATAFED_PYTHON_ENV}" + "python${DATAFED_PYTHON_VERSION}" -m venv "${DATAFED_PYTHON_ENV}" + # Make sure that pip is installed and upgraded + "python${DATAFED_PYTHON_VERSION}" -m ensurepip --upgrade } install_cmake() { if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.cmake_installed-${DATAFED_CMAKE_VERSION}" ]; then - wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz - tar -xzvf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz" >/dev/null 2>&1 - cp -r "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/bin" "${DATAFED_DEPENDENCIES_INSTALL_PATH}" - cp -r "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/share" "${DATAFED_DEPENDENCIES_INSTALL_PATH}" + # Version 3.20 of cmake and onwards starting using all lower case in the package names, previos versions use a + # a capital L in the name. + wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz + tar -xzvf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz" >/dev/null 2>&1 + cp -r "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64/bin" "${DATAFED_DEPENDENCIES_INSTALL_PATH}" + cp -r "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64/share" "${DATAFED_DEPENDENCIES_INSTALL_PATH}" # Cleanup - rm -rf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64" - rm -rf "cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz" + rm -rf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64" + rm -rf "cmake-${DATAFED_CMAKE_VERSION}-linux-x86_64.tar.gz" # Mark cmake as installed touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.cmake_installed-${DATAFED_CMAKE_VERSION}" @@ -149,12 +169,12 @@ install_protobuf() { cd python init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 -m pip install numpy tzdata - LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py build - LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py test + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} -m pip install numpy + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} setup.py build + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python${DATAFED_PYTHON_VERSION} setup.py test # Because we have activaited a venv we don't want to use the --user flag # with the install command - LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py install + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" "python${DATAFED_PYTHON_VERSION}" setup.py install cd ../ # Cleanup build file with root ownership if [ -f build/install_manifest.txt ] @@ -177,14 +197,12 @@ install_libsodium() { # sudo required because of egg file "$SUDO_CMD" rm -rf "${PROJECT_ROOT}/external/libsodium" fi - # Here we are using clone instead of submodule update, because submodule - # requires the .git folder exist and the current folder be considered a repo - # this creates problems in docker because each time a commit is made the - # .git folder contents are changed causing a fresh rebuild of all containers - git clone https://github.com/jedisct1/libsodium.git "${PROJECT_ROOT}/external/libsodium" - cd "${PROJECT_ROOT}/external/libsodium" - git checkout "$DATAFED_LIBSODIUM_VERSION" - ./autogen.sh + # Official documentation for libsodium indicates this is the preferred way to build libsodium. + # Using the git repo directly results in build instability because of additional network calls when running + # autogen.sh. + wget "https://download.libsodium.org/libsodium/releases/libsodium-${DATAFED_LIBSODIUM_VERSION}.tar.gz" -P "${PROJECT_ROOT}/external" + tar -xvzf "${PROJECT_ROOT}/external/libsodium-${DATAFED_LIBSODIUM_VERSION}.tar.gz" -C "${PROJECT_ROOT}/external/" + cd "${PROJECT_ROOT}/external/libsodium-${DATAFED_LIBSODIUM_VERSION}" # Build static ONLY!!!! # Note if zmq detects a shared sodium library it will grab it no matter what # --enable-shared=no must be set here @@ -196,7 +214,7 @@ install_libsodium() { else "$SUDO_CMD" make install fi - + # Mark libsodium as installed touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.libsodium_installed-${DATAFED_LIBSODIUM_VERSION}" cd "$original_dir" diff --git a/scripts/dependency_versions.sh b/scripts/dependency_versions.sh index 2bbdc559c..0ed936fa5 100644 --- a/scripts/dependency_versions.sh +++ b/scripts/dependency_versions.sh @@ -1,9 +1,9 @@ # Versions -DATAFED_CMAKE_VERSION="3.17.5" +DATAFED_CMAKE_VERSION="3.31.6" DATAFED_GLOBUS_VERSION="6.0.31-1" -DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.1.0" -DATAFED_NLOHMANN_JSON_VERSION="3.10.2" +DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.3.0" +DATAFED_NLOHMANN_JSON_VERSION="3.12.0" DATAFED_LIBSODIUM_VERSION="1.0.18" # this version is different from above due to the fact libsodium names its shared library diffrently than the actual api version DATAFED_LIB_LIBSODIUM_VERSION="23.3.0" @@ -14,12 +14,13 @@ DATAFED_LIB_ZMQCPP_VERSION="4.10.0" DATAFED_NVM_VERSION="v0.39.7" DATAFED_NODE_VERSION="v14.21.3" DATAFED_NVM_VERSION="v0.39.7" +DATAFED_PYTHON_VERSION="3.9" # Git tag -DATAFED_PROTOBUF_VERSION="25.2" +DATAFED_PROTOBUF_VERSION="25.7" # Dynamic library extension .so.{DATAFED_FULL_PROTOBUF_VERSION} -DATAFED_DYNAMIC_LIBRARY_PROTOBUF_VERSION="25.2.0" +DATAFED_DYNAMIC_LIBRARY_PROTOBUF_VERSION="25.7.0" # Full version -DATAFED_FULL_PROTOBUF_VERSION="4.25.2" +DATAFED_FULL_PROTOBUF_VERSION="4.25.7" DATAFED_LIBCURL="8.11.0" DATAFED_LIBCURL_URL="https://github.com/curl/curl/releases/download/curl-8_11_0/curl-8.11.0.tar.gz" DATAFED_OPENSSL="1.1.1" diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh index 0ee9da567..21839870e 100755 --- a/scripts/install_authz_dependencies.sh +++ b/scripts/install_authz_dependencies.sh @@ -11,11 +11,13 @@ source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("host" "libtool" "build-essential" "g++" "gcc" "autoconf" - "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config" - "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python3-venv" "libgssapi-krb5-2") + "automake" "make" "git" "pkg-config" + "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv" "libgssapi-krb5-2") pip_packages=("setuptools" "distro" "jwt" "globus_sdk") externals=("cmake" "protobuf" "libsodium" "libzmq" ) +install_python + local_UNIFY=false if [ $# -eq 1 ]; then @@ -47,8 +49,7 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get install -y "${packages[@]}" init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install --upgrade pip - python3 -m pip install "${pip_packages[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/install_client_dependencies.sh b/scripts/install_client_dependencies.sh index 2c475fa4b..0638f87ff 100755 --- a/scripts/install_client_dependencies.sh +++ b/scripts/install_client_dependencies.sh @@ -10,7 +10,10 @@ PROJECT_ROOT=$(realpath "${SOURCE}/..") source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("python3-pkg-resources" "python3-pip" "pkg-config" "python3-venv") +packages=("pkg-config" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv") +pip_packages=("setuptools") + +install_python sudo_command # This script will install all of the dependencies needed by DataFed 1.0 @@ -20,7 +23,8 @@ sudo_command init_python source "${DATAFED_PYTHON_ENV}/bin/activate" -python3 -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt" +"python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" +"python${DATAFED_PYTHON_VERSION}" -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt" install_protobuf cd ~ diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index c79ee55d2..bf7ea6316 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -10,12 +10,14 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "python3-venv") +packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv") pip_packages=("setuptools") # NOTE the order matters here externals=("cmake" "nlohmann_json" "json_schema_validator" "protobuf" "libsodium" "libzmq" "libopenssl" "zlib" "libcurl") +install_python + local_UNIFY=false if [ $# -eq 1 ]; then @@ -46,8 +48,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get install -y "${packages[@]}" init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install --upgrade pip - python3 -m pip install "${pip_packages[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip + "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh index 4ae49ac84..0fbdacdbe 100755 --- a/scripts/install_dependencies.sh +++ b/scripts/install_dependencies.sh @@ -117,7 +117,7 @@ if [ ${#all_pip_packages_array[@]} -gt 0 ]; then echo "DEPENDENCIES (${all_pip_packages_array[@]})" init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install "${all_pip_packages_array[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install "${all_pip_packages_array[@]}" fi all_externals=$(cat "$ext_file_path") diff --git a/scripts/install_docs_dependencies.sh b/scripts/install_docs_dependencies.sh index 06ee7c3de..179382bad 100755 --- a/scripts/install_docs_dependencies.sh +++ b/scripts/install_docs_dependencies.sh @@ -10,7 +10,7 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("g++" "gcc" "make" "python3-pkg-resources" "python3-pip" "pkg-config" "python3-venv") +packages=("g++" "gcc" "make" "pkg-config" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv") externals=("cmake" "protobuf") pip_packages=("setuptools" "sphinx" "sphinx-rtd-theme" "sphinx-autoapi") @@ -44,8 +44,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get install -y "${packages[@]}" init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install --upgrade pip - python3 -m pip install "${pip_packages[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip + "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/install_end_to_end_test_dependencies.sh b/scripts/install_end_to_end_test_dependencies.sh index a4627aceb..ea1e5bddf 100755 --- a/scripts/install_end_to_end_test_dependencies.sh +++ b/scripts/install_end_to_end_test_dependencies.sh @@ -11,7 +11,7 @@ source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "libcurl4-openssl-dev" "wget" - "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "python3-venv" "libssl-dev") + "rapidjson-dev" "libkrb5-dev" "git" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv" "libssl-dev") pip_packages=("setuptools") @@ -47,8 +47,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get install -y "${packages[@]}" init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install --upgrade pip - python3 -m pip install "${pip_packages[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip + "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/install_python_client_dependencies.sh b/scripts/install_python_client_dependencies.sh index a8467a3f0..aa627e2e5 100755 --- a/scripts/install_python_client_dependencies.sh +++ b/scripts/install_python_client_dependencies.sh @@ -17,7 +17,7 @@ sudo_command "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ pkg-config autoconf automake unzip libcurl4-openssl-dev wget \ -rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip python3-venv libssl-dev +rapidjson-dev libkrb5-dev git "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv" libssl-dev cd ~ install_cmake @@ -27,8 +27,8 @@ cd ~ init_python source "${DATAFED_PYTHON_ENV}/bin/activate" -python3 -m pip install --upgrade pip -python3 -m pip install setuptools +"python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip +"python${DATAFED_PYTHON_VERSION}" -m pip install setuptools install_protobuf cd ~ diff --git a/scripts/install_repo_dependencies.sh b/scripts/install_repo_dependencies.sh index 723a3ad81..0bff1c7d8 100755 --- a/scripts/install_repo_dependencies.sh +++ b/scripts/install_repo_dependencies.sh @@ -10,10 +10,12 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python3-pkg-resources" "python3-pip" "python3-venv") +packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python${DATAFED_PYTHON_VERSION}" "python${DATAFED_PYTHON_VERSION}-venv") pip_packages=("setuptools") externals=("cmake" "protobuf" "libsodium" "libzmq") +install_python + local_UNIFY=false if [ $# -eq 1 ]; then @@ -46,8 +48,8 @@ if [[ $local_UNIFY = false ]]; then init_python source "${DATAFED_PYTHON_ENV}/bin/activate" - python3 -m pip install --upgrade pip - python3 -m pip install "${pip_packages[@]}" + "python${DATAFED_PYTHON_VERSION}" -m pip install --upgrade pip + "python${DATAFED_PYTHON_VERSION}" -m pip install "${pip_packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/install_ws_dependencies.sh b/scripts/install_ws_dependencies.sh index 5f124a540..d28e30b5b 100755 --- a/scripts/install_ws_dependencies.sh +++ b/scripts/install_ws_dependencies.sh @@ -11,9 +11,11 @@ source "${PROJECT_ROOT}/scripts/utils.sh" source "${SOURCE}/dependency_versions.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("curl" "python3" "g++" "make" "wget") +packages=("curl" "python${DATAFED_PYTHON_VERSION}" "g++" "make" "wget") externals=("cmake" "nvm" "node" "ws_node_packages") +install_python + Help() { echo "$(basename $0) install web dependencies." diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index 1ad19fa71..6449f28da 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -65,7 +65,7 @@ ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" -RUN apt install -y python3 make g++ +RUN apt install -y make g++ WORKDIR ${DATAFED_DIR}