Merge pull request #2618 from cms-sw/pr-estats-usage

enable usage of external resources utilization for PR tests

Author: smuzaffar

cmsrep.sh

@@ -8,7 +8,7 @@ umask 0002
 export CMS_PYTHON_TO_USE="python"
 if which python3 >/dev/null 2>&1 ; then export CMS_PYTHON_TO_USE="python3" ; fi
 
-#called with $BUILD_OPTS $MULTIARCH_OPTS $ARCH
+#called with $BUILD_OPTS $MULTIARCH_OPTS $ARCH $PKGTOOLS_VER(e.g. 33, 34 etc.)
 function cmsbuild_args()
 {
   arg=""
@@ -17,6 +17,7 @@ function cmsbuild_args()
     for x in $(echo "$1" | tr ',' ' ') ; do
       case $x in
         upload_store ) ;;
+        estats ) ;;
         without:* )    arg="${arg} --build-without=$(echo $x    | sed 's|^without:||;s|:|,|g')" ;;
         system:* )     arg="${arg} --use-system-tools=$(echo $x | sed 's|^system:||;s|:|,|g')" ;;
         microarchs:* ) arg="${arg} --vectorization=$(echo $x    | sed 's|^microarchs:||;s|:|,|g')" ;;

get-external-avg-stats.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+import sys, json
+from es_utils import get_avg_externals_build_stats as get_stats
+
+print(json.dumps(get_stats(arch=sys.argv[1])))

pr_testing/_helper_functions.sh

@@ -104,8 +104,8 @@ function prepare_upload_results (){
           mkdir -p ${LOCAL_LOGDIR}/${xdir}
           mv ${dir}/log ${LOCAL_LOGDIR}/${xdir}/
           [ -e ${dir}/src-logs.tgz ] && mv ${dir}/src-logs.tgz ${LOCAL_LOGDIR}/${xdir}/
-          json=$(basename $(dirname $dir)).json
-          [ -e "${dir}/${json}" ] && mv ${dir}/${json} ${LOCAL_LOGDIR}/${xdir}/
+          pkg=$(basename $(dirname $dir))
+          [ -e "${dir}/${pkg}.json" ] && mv ${dir}/${pkg}*.json ${LOCAL_LOGDIR}/${xdir}/
           [ -e "${dir}/opts.json" ] && mv ${dir}/opts.json ${LOCAL_LOGDIR}/${xdir}/
         done
       popd

pr_testing/test_multiple_prs.sh

@@ -553,7 +553,10 @@ if ${BUILD_EXTERNAL} ; then
     fi
 
     # Build the whole cmssw-tool-conf toolchain
-    CMSBUILD_ARGS="--tag ${PR_NUM}"
+    CMSBUILD_ARGS="--builders 2  --tag ${PR_NUM}"
+    BUILD_OPTS=$(echo $CONFIG_LINE     | tr ';' '\n' | grep "^BUILD_OPTS=" | sed 's|^BUILD_OPTS=||')
+    MULTIARCH_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^MULTIARCH_OPTS=" | sed 's|^MULTIARCH_OPTS=||')
+
     if [ ${PKG_TOOL_VERSION} -gt 31 ] ; then
       CMSBUILD_ARGS="${CMSBUILD_ARGS} --define cmsswdata_version_link  --monitor --log-deps --force-tag --tag hash --delete-build-directory --link-parent-repository"
       if [ "${ALLOW_VERSION_SUFFIX}" = "true" ] ; then
@@ -563,6 +566,13 @@ if ${BUILD_EXTERNAL} ; then
         dbg_pkgs=$(echo "${CONFIG_LINE}" | tr ';' '\n' | grep "^DEBUG_EXTERNALS=" | sed 's|.*=||')
         CMSBUILD_ARGS="${CMSBUILD_ARGS} --define cms_debug_packages=${dbg_pkgs}"
       fi
+      if [ $(echo "${BUILD_OPTS}" | tr ',' '\n' | grep '^estats$') = "estats" ] ; then
+        if [ ${PKG_TOOL_VERSION} -ge 34 ] ; then
+          if ${CMS_BOT_DIR}/get-external-avg-stats.py ${ARCHITECTURE} > ${WORKSPACE}/externals-resource-usage.json ; then
+            CMSBUILD_ARGS="${CMSBUILD_ARGS} --estats ${WORKSPACE}/externals-resource-usage.json --builders ${NCPU}"
+          fi
+        fi
+      fi
     fi
     if [ $(grep 'upload-package-store-s3' pkgtools/cmsBuild | wc -l) -gt 0 ] ; then
       [ "${CMSBOT_SET_ENV_NO_PACKAGE_STORE}" = "true" ] && UPLOAD_TO_PACKAGE_STORE=false
@@ -572,13 +582,10 @@ if ${BUILD_EXTERNAL} ; then
         CMSBUILD_ARGS="${CMSBUILD_ARGS} --no-package-store"
       fi
     fi
-    #Process cmsdist Build options
-    BUILD_OPTS=$(echo $CONFIG_LINE     | tr ';' '\n' | grep "^BUILD_OPTS=" | sed 's|^BUILD_OPTS=||')
-    MULTIARCH_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^MULTIARCH_OPTS=" | sed 's|^MULTIARCH_OPTS=||')
 
     PKGS="cms-common cms-git-tools cmssw-tool-conf"
     COMPILATION_CMD="PYTHONPATH= ./pkgtools/cmsBuild --server http://${CMSREP_IB_SERVER}/cgi-bin/cmspkg --upload-server ${CMSREP_IB_SERVER} \
-        ${CMSBUILD_ARGS} --builders 2 -i $WORKSPACE/$BUILD_DIR $REF_REPO \
+        ${CMSBUILD_ARGS} -i $WORKSPACE/$BUILD_DIR $REF_REPO \
         $SOURCE_FLAG --arch $ARCHITECTURE -j ${NCPU} $(cmsbuild_args "${BUILD_OPTS}" "${MULTIARCH_OPTS}" "${ARCHITECTURE}")"
     PR_EXTERNAL_REPO="PR_$(echo ${RPM_UPLOAD_REPO}_${CMSSW_QUEUE}_${ARCHITECTURE} | md5sum | sed 's| .*||' | tail -c 9)"
     if [ -e cmsdist/cmssw-tool-conf.spec ] ; then