From a4d95cf0841e99a9df2f6768895b590c8bd2aeed Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 7 Sep 2023 23:33:02 +0200 Subject: [PATCH] bring logging utils --- .scripts/build_steps.sh | 33 +++++++++++++++++++++--------- .scripts/logging_utils.sh | 35 ++++++++++++++++++++++++++++++++ .scripts/run_docker_build.sh | 11 ++++++++++ .scripts/run_osx_build.sh | 39 ++++++++++++++++++------------------ 4 files changed, 88 insertions(+), 30 deletions(-) create mode 100644 .scripts/logging_utils.sh diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 8c40aee1f0ba8..f256153753856 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -6,7 +6,17 @@ # benefit from the improvement. set -xeuo pipefail + +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/staged-recipes}" +source "${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh" + +# This closes the matching `startgroup` on `run_docker_build.sh` +( endgroup "Start Docker" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + export PYTHONUNBUFFERED=1 +export CI_SUPPORT="/home/conda/staged-recipes-copy/.ci_support" cat >~/.condarc < /dev/null +pushd "${FEEDSTOCK_ROOT}/recipes" > /dev/null if [ "${AZURE}" == "True" ]; then git fetch --force origin main:main fi @@ -44,13 +54,9 @@ git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf ~/staged-recipes-c popd > /dev/null -# Make sure build_artifacts is a valid channel -conda index /home/conda/staged-recipes/build_artifacts -conda install --quiet --file /home/conda/staged-recipes/.ci_support/requirements.txt +conda install --quiet --file ${FEEDSTOCK_ROOT}/.ci_support/requirements.txt -export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/staged-recipes}" -export CI_SUPPORT="/home/conda/staged-recipes-copy/.ci_support" setup_conda_rc "${FEEDSTOCK_ROOT}" "/home/conda/staged-recipes-copy/recipes" "${CI_SUPPORT}/${CONFIG}.yaml" source run_conda_forge_build_setup @@ -59,6 +65,13 @@ find ~/staged-recipes-copy/recipes -mindepth 2 -maxdepth 2 -type f -name "yum_re | xargs -n1 cat | { grep -v -e "^#" -e "^$" || test $? == 1; } | \ xargs -r /usr/bin/sudo -n yum install -y +# Make sure build_artifacts is a valid channel +conda index ${FEEDSTOCK_ROOT}/build_artifacts + +( endgroup "Configuring conda" ) 2> /dev/null + python ${CI_SUPPORT}/build_all.py -touch "/home/conda/staged-recipes/build_artifacts/conda-forge-build-done" +( startgroup "Final checks" ) 2> /dev/null + +touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done" diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh new file mode 100644 index 0000000000000..57bc95c2421b4 --- /dev/null +++ b/.scripts/logging_utils.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Provide a unified interface for the different logging +# utilities CI providers offer. If unavailable, provide +# a compatible fallback (e.g. bare `echo xxxxxx`). + +function startgroup { + # Start a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[group]$1";; + travis ) + echo "$1" + echo -en 'travis_fold:start:'"${1// /}"'\\r';; + github_actions ) + echo "::group::$1";; + * ) + echo "$1";; + esac +} 2> /dev/null + +function endgroup { + # End a foldable group of log lines + # Pass a single argument, quoted + + case ${CI:-} in + azure ) + echo "##[endgroup]";; + travis ) + echo -en 'travis_fold:end:'"${1// /}"'\\r';; + github_actions ) + echo "::endgroup::";; + esac +} 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 36befe29d93df..96572c0031a59 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -2,6 +2,10 @@ # NOTE: This script has been adapted from content generated by github.com/conda-forge/conda-smithy +source .scripts/logging_utils.sh + +( startgroup "Configure Docker" ) 2> /dev/null + REPO_ROOT=$(cd "$(dirname "$0")/.."; pwd;) ARTIFACTS="$REPO_ROOT/build_artifacts" THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" @@ -43,6 +47,10 @@ DOCKER_RUN_ARGS="-it" if [ "${AZURE}" == "True" ]; then DOCKER_RUN_ARGS="" fi +( endgroup "Configure Docker" ) 2> /dev/null + +( startgroup "Start Docker" ) 2> /dev/null +# this group is closed in build_steps.sh docker pull "${DOCKER_IMAGE}" docker run ${DOCKER_RUN_ARGS} \ @@ -59,3 +67,6 @@ docker run ${DOCKER_RUN_ARGS} \ # verify that the end of the script was reached test -f "$DONE_CANARY" + +# This closes the last group opened in `build_steps.sh` +( endgroup "Final checks" ) 2> /dev/null diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index c949ea0e1d468..9959b3ddf1c58 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -2,30 +2,33 @@ set -x -echo -e "\n\nInstalling a fresh version of Miniforge." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:install_miniforge\\r' -fi +source .scripts/logging_utils.sh + +( startgroup "Ensuring Miniforge" ) 2> /dev/null + MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" -curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" -bash $MiniFORGE_FILE -b -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:end:install_miniforge\\r' -fi +MINIFORGE_ROOT="${MINIFORGE_ROOT:-${HOME}/Miniforge3}" -echo -e "\n\nConfiguring conda." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:configure_conda\\r' +if [[ -d "${MINIFORGE_ROOT}" ]]; then + echo "Miniforge already installed at ${MINIFORGE_ROOT}." +else + echo "Installing Miniforge" + curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" + bash $MINIFORGE_FILE -bp "${MINIFORGE_ROOT}" fi +( endgroup "Ensuring Miniforge" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + cat >~/.condarc < /dev/null @@ -68,5 +65,7 @@ git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf {} && echo Removin popd > /dev/null echo "" +( endgroup "Configuring conda" ) 2> /dev/null + # We just want to build all of the recipes. python .ci_support/build_all.py