add extra merge commits for cmsdist too

pr_testing/get-merged-prs.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 from __future__ import print_function
 from os import environ
-from os.path import dirname,basename,abspath
-from json import dumps, dump
+from os.path import dirname,basename,abspath,join
+from json import dumps, dump, load
 from optparse import OptionParser
 import sys
 sys.path.append(dirname(dirname(abspath(__file__)))) 
@@ -14,6 +14,8 @@
 parser.add_option( "-g", "--git-directory", dest="git_dir" , action="store" , help=".git directory, default is CMSSW_BASE/src/.git", default=None)
 parser.add_option( "-c", "--cache-directory", dest="cache_dir" , action="store" , help="Path to cms-prs cache directory", default=None)
 parser.add_option( "-o", "--out-file", dest="out_file" , action="store" , help="Outpu json file name", default=None)
+parser.add_option( "-r", "--repository", dest="repository" , action="store" , help="Repository e.g. cms-sw/cmssw or cms-sw/cmsdist", default="cms-sw/cmssw")
+parser.add_option( "-i", "--ignore-prs", dest="ignore" , action="store" , help="Comma separated list of PRs to ignore", default="")
 opts, args = parser.parse_args( )
 if len( args ) != 0:
   parser.print_help()
@@ -23,11 +25,18 @@
 if not opts.git_dir:
   opts.git_dir = environ['CMSSW_BASE']+"/src/.git"
 if not opts.cache_dir:
-  parser.error( "Please pass -c|--cache-directory /path/to/cms-prs/cms-sw/<repository>" )
+  parser.error( "Please pass -c|--cache-directory /path/to/cms-prs" )
 
-repo = basename(opts.cache_dir)
-opts.cache_dir = dirname(dirname(opts.cache_dir))
-prs = get_merge_prs(opts.start_tag, opts.end_tag, opts.git_dir,opts.cache_dir,{},repo)
+prs = {}
+if opts.out_file:
+  with open(opts.out_file) as ref:
+    prs = load(ref)
+prs[opts.repository] = get_merge_prs(opts.start_tag, opts.end_tag, opts.git_dir,opts.cache_dir,{},basename(opts.repository))
+for ignore in [int(i) for i in opts.ignore.split(",") if i]:
+  if ignore in prs[opts.repository]:
+    del prs[opts.repository][ignore]
+if not prs[opts.repository]:
+  del prs[opts.repository]
 if opts.out_file:
   with open(opts.out_file,"w") as ref:
     dump(prs, ref,sort_keys=True, indent=4, separators=(',', ': '))

pr_testing/test_multiple_prs.sh

@@ -528,9 +528,11 @@ git config --global --replace-all merge.renamelimit 2500 || true
 
 GIT_MERGE_RESULT_FILE=$WORKSPACE/git-merge-result
 RECENT_COMMITS_FILE=$WORKSPACE/git-recent-commits.json
+RECENT_COMMITS_LOG_FILE=$WORKSPACE/git-log-recent-commits
 echo '{}' > $RECENT_COMMITS_FILE
 # use the branch name if necesary
 touch $WORKSPACE/changed-files
+if [ ! -d $WORKSPACE/cms-prs ]  ; then git clone --depth 1 git@github.com:cms-sw/cms-prs $WORKSPACE/cms-prs ; fi
 if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
   # this is to test several pull requests at the same time
   for PR in $( echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#"); do
@@ -568,20 +570,29 @@ if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
   #############################################
   # Check if there are unwanted commits that came with the merge.
   ############################################
-  RECENT_COMMITS_LOG_FILE=$WORKSPACE/git-log-recent-commits
+  merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#" | sed 's|.*#||' | tr '\n' ',')
+  $SCRIPTPATH/get-merged-prs.py -r cms-sw/cmssw -i "${merged_prs}" -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE
+  echo "##### CMSSW Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
+  git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
 
-  if [ ! -d $WORKSPACE/cms-prs ]  ; then
-    git clone --depth 1 git@github.com:cms-sw/cms-prs $WORKSPACE/cms-prs
-  fi
-  $SCRIPTPATH/get-merged-prs.py -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs/cms-sw/cmssw -o $RECENT_COMMITS_FILE
-  $CMS_BOT_DIR/report-pull-request-results MERGE_COMMITS --recent-merges $RECENT_COMMITS_FILE --report-url ${PR_RESULT_URL} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS}
-  git log ${CMSSW_IB}..HEAD --merges 2>&1      | tee -a $RECENT_COMMITS_LOG_FILE
   if [ $DO_MB_COMPARISON -a $(grep 'Geometry' $WORKSPACE/changed-files | wc -l) -gt 0 ] ; then
     has_jenkins_artifacts material-budget/$CMSSW_IB/$SCRAM_ARCH/Images || DO_MB_COMPARISON=false
   else
     DO_MB_COMPARISON=false
   fi
 fi
+if ${BUILD_EXTERNAL} ; then
+  pushd $WORKSPACE/cmsdist
+    CMSDIST_REL_TAG=$(git tag | grep '^'ALL/${CMSSW_VERSION}/${SCRAM_ARCH}'$' || true)
+    if [ "${CMSDIST_REL_TAG}" != "" ] ; then
+      merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmsdist#" | sed 's|.*#||' | tr '\n' ',')
+      $SCRIPTPATH/get-merged-prs.py -r cms-sw/cmsdist -i "${merged_prs}" -s ${CMSDIST_REL_TAG} -e HEAD -g $WORKSPACE/cmsdist/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE
+      echo "##### CMSDIST Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
+      git log ${CMSDIST_REL_TAG}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
+    fi
+  popd
+fi
+$CMS_BOT_DIR/report-pull-request-results MERGE_COMMITS --recent-merges $RECENT_COMMITS_FILE --report-url ${PR_RESULT_URL} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS}
 
 # Don't do the following if we are only testing CMSDIST PR
 if [ "X$CMSDIST_ONLY" == Xfalse ]; then

report-pull-request-results.py

@@ -179,7 +179,8 @@ def get_recent_merges_message():
   if options.recent_merges_file:
     extra_msg = []
     json_obj = json.load(open(options.recent_merges_file))
-    for pr in json_obj: extra_msg.append(" - #%s @%s: %s" % (pr, json_obj[pr]['author'], json_obj[pr]['title']))
+    for r in json_obj:
+      for pr in json_obj[r]: extra_msg.append(" - %s#%s @%s: %s" % (r, pr, json_obj[r][pr]['author'], json_obj[r][pr]['title']))
 
     if extra_msg:
       message += '\n\nThe following merge commits were also included on top of IB + this PR '\