diff --git a/pr_testing/get-merged-prs.py b/pr_testing/get-merged-prs.py index 51f3fb65c0ca..3652b3808284 100755 --- a/pr_testing/get-merged-prs.py +++ b/pr_testing/get-merged-prs.py @@ -1,8 +1,8 @@ #!/usr/bin/env python from __future__ import print_function from os import environ -from os.path import dirname,basename,abspath -from json import dumps, dump +from os.path import dirname,basename,abspath,join +from json import dumps, dump, load from optparse import OptionParser import sys sys.path.append(dirname(dirname(abspath(__file__)))) @@ -14,6 +14,8 @@ parser.add_option( "-g", "--git-directory", dest="git_dir" , action="store" , help=".git directory, default is CMSSW_BASE/src/.git", default=None) parser.add_option( "-c", "--cache-directory", dest="cache_dir" , action="store" , help="Path to cms-prs cache directory", default=None) parser.add_option( "-o", "--out-file", dest="out_file" , action="store" , help="Outpu json file name", default=None) +parser.add_option( "-r", "--repository", dest="repository" , action="store" , help="Repository e.g. cms-sw/cmssw or cms-sw/cmsdist", default="cms-sw/cmssw") +parser.add_option( "-i", "--ignore-prs", dest="ignore" , action="store" , help="Comma separated list of PRs to ignore", default="") opts, args = parser.parse_args( ) if len( args ) != 0: parser.print_help() @@ -23,11 +25,18 @@ if not opts.git_dir: opts.git_dir = environ['CMSSW_BASE']+"/src/.git" if not opts.cache_dir: - parser.error( "Please pass -c|--cache-directory /path/to/cms-prs/cms-sw/" ) + parser.error( "Please pass -c|--cache-directory /path/to/cms-prs" ) -repo = basename(opts.cache_dir) -opts.cache_dir = dirname(dirname(opts.cache_dir)) -prs = get_merge_prs(opts.start_tag, opts.end_tag, opts.git_dir,opts.cache_dir,{},repo) +prs = {} +if opts.out_file: + with open(opts.out_file) as ref: + prs = load(ref) +prs[opts.repository] = get_merge_prs(opts.start_tag, opts.end_tag, opts.git_dir,opts.cache_dir,{},basename(opts.repository)) +for ignore in [int(i) for i in opts.ignore.split(",") if i]: + if ignore in prs[opts.repository]: + del prs[opts.repository][ignore] +if not prs[opts.repository]: + del prs[opts.repository] if opts.out_file: with open(opts.out_file,"w") as ref: dump(prs, ref,sort_keys=True, indent=4, separators=(',', ': ')) diff --git a/pr_testing/test_multiple_prs.sh b/pr_testing/test_multiple_prs.sh index 569ad3f134ae..18fe46ede0d0 100755 --- a/pr_testing/test_multiple_prs.sh +++ b/pr_testing/test_multiple_prs.sh @@ -528,9 +528,11 @@ git config --global --replace-all merge.renamelimit 2500 || true GIT_MERGE_RESULT_FILE=$WORKSPACE/git-merge-result RECENT_COMMITS_FILE=$WORKSPACE/git-recent-commits.json +RECENT_COMMITS_LOG_FILE=$WORKSPACE/git-log-recent-commits echo '{}' > $RECENT_COMMITS_FILE # use the branch name if necesary touch $WORKSPACE/changed-files +if [ ! -d $WORKSPACE/cms-prs ] ; then git clone --depth 1 git@github.com:cms-sw/cms-prs $WORKSPACE/cms-prs ; fi if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified # # this is to test several pull requests at the same time for PR in $( echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#"); do @@ -568,20 +570,29 @@ if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified # ############################################# # Check if there are unwanted commits that came with the merge. ############################################ - RECENT_COMMITS_LOG_FILE=$WORKSPACE/git-log-recent-commits + merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#" | sed 's|.*#||' | tr '\n' ',') + $SCRIPTPATH/get-merged-prs.py -r cms-sw/cmssw -i "${merged_prs}" -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE + echo "##### CMSSW Extra merges #####" >> $RECENT_COMMITS_LOG_FILE + git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE - if [ ! -d $WORKSPACE/cms-prs ] ; then - git clone --depth 1 git@github.com:cms-sw/cms-prs $WORKSPACE/cms-prs - fi - $SCRIPTPATH/get-merged-prs.py -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs/cms-sw/cmssw -o $RECENT_COMMITS_FILE - $CMS_BOT_DIR/report-pull-request-results MERGE_COMMITS --recent-merges $RECENT_COMMITS_FILE --report-url ${PR_RESULT_URL} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS} - git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE if [ $DO_MB_COMPARISON -a $(grep 'Geometry' $WORKSPACE/changed-files | wc -l) -gt 0 ] ; then has_jenkins_artifacts material-budget/$CMSSW_IB/$SCRAM_ARCH/Images || DO_MB_COMPARISON=false else DO_MB_COMPARISON=false fi fi +if ${BUILD_EXTERNAL} ; then + pushd $WORKSPACE/cmsdist + CMSDIST_REL_TAG=$(git tag | grep '^'ALL/${CMSSW_VERSION}/${SCRAM_ARCH}'$' || true) + if [ "${CMSDIST_REL_TAG}" != "" ] ; then + merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmsdist#" | sed 's|.*#||' | tr '\n' ',') + $SCRIPTPATH/get-merged-prs.py -r cms-sw/cmsdist -i "${merged_prs}" -s ${CMSDIST_REL_TAG} -e HEAD -g $WORKSPACE/cmsdist/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE + echo "##### CMSDIST Extra merges #####" >> $RECENT_COMMITS_LOG_FILE + git log ${CMSDIST_REL_TAG}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE + fi + popd +fi +$CMS_BOT_DIR/report-pull-request-results MERGE_COMMITS --recent-merges $RECENT_COMMITS_FILE --report-url ${PR_RESULT_URL} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS} # Don't do the following if we are only testing CMSDIST PR if [ "X$CMSDIST_ONLY" == Xfalse ]; then diff --git a/report-pull-request-results.py b/report-pull-request-results.py index 204945e6beed..eafcd6b9c813 100755 --- a/report-pull-request-results.py +++ b/report-pull-request-results.py @@ -179,7 +179,8 @@ def get_recent_merges_message(): if options.recent_merges_file: extra_msg = [] json_obj = json.load(open(options.recent_merges_file)) - for pr in json_obj: extra_msg.append(" - #%s @%s: %s" % (pr, json_obj[pr]['author'], json_obj[pr]['title'])) + for r in json_obj: + for pr in json_obj[r]: extra_msg.append(" - %s#%s @%s: %s" % (r, pr, json_obj[r][pr]['author'], json_obj[r][pr]['title'])) if extra_msg: message += '\n\nThe following merge commits were also included on top of IB + this PR '\