Skip to content

enable usage of external resources utilization for PR tests #2618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 19, 2025
Merged

enable usage of external resources utilization for PR tests #2618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6f0f01b
enable usage of external resources utilization for PR tests
smuzaffar Nov 17, 2025
e975ab5
typo
smuzaffar Nov 17, 2025
273eb6f
Update config.map
smuzaffar Nov 17, 2025
62642ca
Upload newly generated resource monitor files
smuzaffar Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmsrep.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ umask 0002
export CMS_PYTHON_TO_USE="python"
if which python3 >/dev/null 2>&1 ; then export CMS_PYTHON_TO_USE="python3" ; fi

#called with $BUILD_OPTS $MULTIARCH_OPTS $ARCH
#called with $BUILD_OPTS $MULTIARCH_OPTS $ARCH $PKGTOOLS_VER(e.g. 33, 34 etc.)
function cmsbuild_args()
{
arg=""
Expand All @@ -17,6 +17,7 @@ function cmsbuild_args()
for x in $(echo "$1" | tr ',' ' ') ; do
case $x in
upload_store ) ;;
estats ) ;;
without:* ) arg="${arg} --build-without=$(echo $x | sed 's|^without:||;s|:|,|g')" ;;
system:* ) arg="${arg} --use-system-tools=$(echo $x | sed 's|^system:||;s|:|,|g')" ;;
microarchs:* ) arg="${arg} --vectorization=$(echo $x | sed 's|^microarchs:||;s|:|,|g')" ;;
Expand Down
5 changes: 5 additions & 0 deletions get-external-avg-stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/usr/bin/env python3
import sys, json
from es_utils import get_avg_externals_build_stats as get_stats

print(json.dumps(get_stats(arch=sys.argv[1])))
4 changes: 2 additions & 2 deletions pr_testing/_helper_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ function prepare_upload_results (){
mkdir -p ${LOCAL_LOGDIR}/${xdir}
mv ${dir}/log ${LOCAL_LOGDIR}/${xdir}/
[ -e ${dir}/src-logs.tgz ] && mv ${dir}/src-logs.tgz ${LOCAL_LOGDIR}/${xdir}/
json=$(basename $(dirname $dir)).json
[ -e "${dir}/${json}" ] && mv ${dir}/${json} ${LOCAL_LOGDIR}/${xdir}/
pkg=$(basename $(dirname $dir))
[ -e "${dir}/${pkg}.json" ] && mv ${dir}/${pkg}*.json ${LOCAL_LOGDIR}/${xdir}/
[ -e "${dir}/opts.json" ] && mv ${dir}/opts.json ${LOCAL_LOGDIR}/${xdir}/
done
popd
Expand Down
17 changes: 12 additions & 5 deletions pr_testing/test_multiple_prs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,10 @@ if ${BUILD_EXTERNAL} ; then
fi

# Build the whole cmssw-tool-conf toolchain
CMSBUILD_ARGS="--tag ${PR_NUM}"
CMSBUILD_ARGS="--builders 2 --tag ${PR_NUM}"
BUILD_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^BUILD_OPTS=" | sed 's|^BUILD_OPTS=||')
MULTIARCH_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^MULTIARCH_OPTS=" | sed 's|^MULTIARCH_OPTS=||')

if [ ${PKG_TOOL_VERSION} -gt 31 ] ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --define cmsswdata_version_link --monitor --log-deps --force-tag --tag hash --delete-build-directory --link-parent-repository"
if [ "${ALLOW_VERSION_SUFFIX}" = "true" ] ; then
Expand All @@ -563,6 +566,13 @@ if ${BUILD_EXTERNAL} ; then
dbg_pkgs=$(echo "${CONFIG_LINE}" | tr ';' '\n' | grep "^DEBUG_EXTERNALS=" | sed 's|.*=||')
CMSBUILD_ARGS="${CMSBUILD_ARGS} --define cms_debug_packages=${dbg_pkgs}"
fi
if [ $(echo "${BUILD_OPTS}" | tr ',' '\n' | grep '^estats$') = "estats" ] ; then
if [ ${PKG_TOOL_VERSION} -ge 34 ] ; then
if ${CMS_BOT_DIR}/get-external-avg-stats.py ${ARCHITECTURE} > ${WORKSPACE}/externals-resource-usage.json ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --estats ${WORKSPACE}/externals-resource-usage.json --builders ${NCPU}"
fi
fi
fi
fi
if [ $(grep 'upload-package-store-s3' pkgtools/cmsBuild | wc -l) -gt 0 ] ; then
[ "${CMSBOT_SET_ENV_NO_PACKAGE_STORE}" = "true" ] && UPLOAD_TO_PACKAGE_STORE=false
Expand All @@ -572,13 +582,10 @@ if ${BUILD_EXTERNAL} ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --no-package-store"
fi
fi
#Process cmsdist Build options
BUILD_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^BUILD_OPTS=" | sed 's|^BUILD_OPTS=||')
MULTIARCH_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^MULTIARCH_OPTS=" | sed 's|^MULTIARCH_OPTS=||')

PKGS="cms-common cms-git-tools cmssw-tool-conf"
COMPILATION_CMD="PYTHONPATH= ./pkgtools/cmsBuild --server http://${CMSREP_IB_SERVER}/cgi-bin/cmspkg --upload-server ${CMSREP_IB_SERVER} \
${CMSBUILD_ARGS} --builders 2 -i $WORKSPACE/$BUILD_DIR $REF_REPO \
${CMSBUILD_ARGS} -i $WORKSPACE/$BUILD_DIR $REF_REPO \
$SOURCE_FLAG --arch $ARCHITECTURE -j ${NCPU} $(cmsbuild_args "${BUILD_OPTS}" "${MULTIARCH_OPTS}" "${ARCHITECTURE}")"
PR_EXTERNAL_REPO="PR_$(echo ${RPM_UPLOAD_REPO}_${CMSSW_QUEUE}_${ARCHITECTURE} | md5sum | sed 's| .*||' | tail -c 9)"
if [ -e cmsdist/cmssw-tool-conf.spec ] ; then
Expand Down