Skip to content

Commit 659cb94

Browse files
iarspideriarspider
committed
PR Testing: check for warnings also in files not directly touched by PR
1 parent cd182e3 commit 659cb94

1 file changed

Lines changed: 93 additions & 54 deletions

File tree

‎pr_testing/test_multiple_prs.sh‎

Lines changed: 93 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,37 @@ function get_pr_relval_args() {
4444
echo "${WF_ARGS}"
4545
}
4646

47+
# Function to extract filenames by headername and append to changed-files.txt atomically
48+
function extract_filenames() {
49+
local headername="$1"
50+
local input_file="./etc/dependencies/usedby.out"
51+
local output_file="$WORKSPACE/extra-changed-files"
52+
[ ! -e $output_file ] && touch $output_file
53+
54+
# Extract lines starting with headername, split them, and append each filename to the temp file
55+
grep "^$headername" "$input_file" | while read -r line; do
56+
# Split the line into an array
57+
IFS=' ' read -r -a array <<< "$line"
58+
59+
# Loop through each element after the first (which is the headername)
60+
for filename in "${array[@]:1}"; do
61+
echo "$filename" >> "$output_file"
62+
done
63+
done
64+
}
65+
66+
# Function to process each line in $WORKSPACE/changed-files
67+
function process_changed_files() {
68+
# Iterate over each line in $WORKSPACE/changed-files
69+
while IFS= read -r headername; do
70+
# Call the function to extract filenames and append them atomically
71+
extract_filenames "$headername"
72+
done < "$WORKSPACE/changed-files"
73+
local temp_file=$(mktemp)
74+
sort -u $WORKSPACE/changed-files $WORKSPACE/extra-changed-files > $temp_file
75+
mv $temp_file $WORKSPACE/changed-files
76+
}
77+
4778
# Constants
4879
echo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} || true
4980
ls ${LD_LIBRARY_PATH} || true
@@ -824,8 +855,6 @@ if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
824855
exit 0
825856
fi
826857

827-
git diff --name-only $CMSSW_VERSION > $WORKSPACE/changed-files
828-
829858
# look for any other error in general
830859
if ! grep "ALL_OK" $GIT_MERGE_RESULT_FILE; then
831860
echo "There was an issue with git-cms-merge-topic you can see the log here: ${PR_RESULT_URL}/git-merge-result" > ${RESULTS_DIR}/10-report.res
@@ -897,57 +926,6 @@ if [ "X$CMSDIST_ONLY" == Xfalse ]; then
897926
git log --oneline --merges ${CMSSW_VERSION}..
898927
fi
899928

900-
# #############################################
901-
# test compilation with Clang
902-
# ############################################
903-
echo 'test clang compilation'
904-
905-
NEED_CLANG_TEST=false
906-
if cat $CONFIG_MAP | grep $CMSSW_QUEUE | grep PRS_TEST_CLANG= | grep SCRAM_ARCH=$ARCHITECTURE; then
907-
NEED_CLANG_TEST=true
908-
fi
909-
910-
if [ "X$TEST_CLANG_COMPILATION" = Xtrue -a $NEED_CLANG_TEST = true -a "X$CMSSW_PR" != X -a "$SKIP_STATIC_CHECKS" = "false" ]; then
911-
#first, add the command to the log
912-
CLANG_USER_CMD="USER_CUDA_FLAGS='--expt-relaxed-constexpr' USER_CXXFLAGS='-Wno-register -fsyntax-only' scram build -k -j ${NCPU2} COMPILER='llvm compile'"
913-
CLANG_CMD="scram b vclean && ${CLANG_USER_CMD} BUILD_LOG=yes"
914-
echo $CLANG_USER_CMD > $WORKSPACE/buildClang.log
915-
916-
(eval $CLANG_CMD && echo 'ALL_OK') >>$WORKSPACE/buildClang.log 2>&1 || true
917-
(eval ${ANALOG_CMD}) >>$WORKSPACE/buildClang.log 2>&1 || true
918-
919-
TEST_ERRORS=`grep -E "^gmake: .* Error [0-9]" $WORKSPACE/buildClang.log` || true
920-
GENERAL_ERRORS=`grep "ALL_OK" $WORKSPACE/buildClang.log` || true
921-
get_compilation_warnings $WORKSPACE/buildClang.log > $WORKSPACE/all-warnings-clang.log
922-
for i in $(get_warnings_files $WORKSPACE/all-warnings-clang.log) ; do
923-
echo $i >> $WORKSPACE/clang-new-warnings.log
924-
grep ": warning: " $WORKSPACE/all-warnings-clang.log | grep "/$i" >> $WORKSPACE/clang-new-warnings.log
925-
done
926-
if [ -e $WORKSPACE/clang-new-warnings.log ] ; then
927-
echo 'CLANG_NEW_WARNINGS;ERROR,Clang Warnings to fix,See Log,clang-new-warnings.log' >> ${RESULTS_DIR}/clang.txt
928-
if $IS_DEV_BRANCH && [ $(echo "$IGNORE_BOT_TESTS" | tr ',' '\n' | grep '^CLANG-WARNINGS$' | wc -l) -eq 0 ] ; then
929-
RUN_TESTS=false
930-
ALL_OK=false
931-
CLANG_BUILD_OK=false
932-
fi
933-
fi
934-
935-
if [ -d ${BUILD_LOG_DIR}/html ] ; then
936-
mv ${BUILD_LOG_DIR}/html $WORKSPACE/clang-logs
937-
echo 'CLANG_LOG;OK,Clang warnings summary,See Log,clang-logs' >> ${RESULTS_DIR}/clang.txt
938-
fi
939-
if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
940-
echo "Errors when testing compilation with clang"
941-
echo 'CLANG_COMPILATION_RESULTS;ERROR,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
942-
RUN_TESTS=false
943-
ALL_OK=false
944-
CLANG_BUILD_OK=false
945-
else
946-
echo "the clang compilation had no errors/warnings!!"
947-
echo 'CLANG_COMPILATION_RESULTS;OK,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
948-
fi
949-
fi
950-
951929
#Do QA checks
952930
#Code Rules
953931
QA_RES="NOTRUN"
@@ -1141,6 +1119,17 @@ GENERAL_ERRORS=`grep "ALL_OK" $WORKSPACE/build.log` || true
11411119

11421120
rm -f $WORKSPACE/deprecated-warnings.log
11431121
get_compilation_warnings $WORKSPACE/build.log > $WORKSPACE/all-warnings.log
1122+
1123+
git diff --name-only $CMSSW_VERSION > $WORKSPACE/changed-files
1124+
pushd ..
1125+
scram b echo_SCRAM_TOOL_HOME
1126+
test -e config/SCRAM/findDependencies.py
1127+
SCRAM_TOOL_HOME=`scram b echo_SCRAM_TOOL_HOME 2>/dev/null | tail -1 | cut -d' ' -f3`
1128+
mkdir -p etc/dependencies
1129+
SCRAM_TOOL_HOME=$SCRAM_TOOL_HOME ./config/SCRAM/findDependencies.py -rel `pwd` -arch ${SCRAM_ARCH}
1130+
process_changed_files
1131+
popd
1132+
11441133
for i in $(get_warnings_files $WORKSPACE/all-warnings.log) ; do
11451134
echo $i > $WORKSPACE/warning.log
11461135
grep ": warning: " $WORKSPACE/all-warnings.log | grep "/$i" >> $WORKSPACE/warning.log
@@ -1166,7 +1155,6 @@ if [ -e $WORKSPACE/deprecated-warnings.log ] ; then
11661155
echo 'BUILD_DEPRECATED_WARNINGS;ERROR,CMS Deprecated Warnings,See Log,deprecated-warnings.log' >> ${RESULTS_DIR}/buildrules.txt
11671156
echo "**CMS deprecated warnings**: $(cat ${WORKSPACE}/deprecated-warnings.log | grep 'Wdeprecated-declarations' | wc -l) CMS deprecated warnings found, see [summary page](${PR_RESULT_URL}/deprecated-warnings.log) for details." >> ${RESULTS_DIR}/09-report.res
11681157
fi
1169-
11701158
BUILD_LOG_RES="ERROR"
11711159
if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
11721160
echo "Errors when building"
@@ -1200,6 +1188,57 @@ fi
12001188
echo "BUILD_LOG;${BUILD_LOG_RES},Compilation warnings summary,See Logs,build-logs" >> ${RESULTS_DIR}/build.txt
12011189
mark_commit_status_all_prs '' 'pending' -u "${BUILD_URL}" -d "Running tests" || true
12021190

1191+
# #############################################
1192+
# test compilation with Clang
1193+
# ############################################
1194+
echo 'test clang compilation'
1195+
1196+
NEED_CLANG_TEST=false
1197+
if cat $CONFIG_MAP | grep $CMSSW_QUEUE | grep PRS_TEST_CLANG= | grep SCRAM_ARCH=$ARCHITECTURE; then
1198+
NEED_CLANG_TEST=true
1199+
fi
1200+
1201+
if [ "X$TEST_CLANG_COMPILATION" = Xtrue -a $NEED_CLANG_TEST = true -a "X$CMSSW_PR" != X -a "$SKIP_STATIC_CHECKS" = "false" ]; then
1202+
#first, add the command to the log
1203+
CLANG_USER_CMD="USER_CUDA_FLAGS='--expt-relaxed-constexpr' USER_CXXFLAGS='-Wno-register -fsyntax-only' scram build -k -j ${NCPU2} COMPILER='llvm compile'"
1204+
CLANG_CMD="scram b vclean && ${CLANG_USER_CMD} BUILD_LOG=yes"
1205+
echo $CLANG_USER_CMD > $WORKSPACE/buildClang.log
1206+
1207+
(eval $CLANG_CMD && echo 'ALL_OK') >>$WORKSPACE/buildClang.log 2>&1 || true
1208+
(eval ${ANALOG_CMD}) >>$WORKSPACE/buildClang.log 2>&1 || true
1209+
1210+
TEST_ERRORS=`grep -E "^gmake: .* Error [0-9]" $WORKSPACE/buildClang.log` || true
1211+
GENERAL_ERRORS=`grep "ALL_OK" $WORKSPACE/buildClang.log` || true
1212+
get_compilation_warnings $WORKSPACE/buildClang.log > $WORKSPACE/all-warnings-clang.log
1213+
for i in $(get_warnings_files $WORKSPACE/all-warnings-clang.log) ; do
1214+
echo $i >> $WORKSPACE/clang-new-warnings.log
1215+
grep ": warning: " $WORKSPACE/all-warnings-clang.log | grep "/$i" >> $WORKSPACE/clang-new-warnings.log
1216+
done
1217+
if [ -e $WORKSPACE/clang-new-warnings.log ] ; then
1218+
echo 'CLANG_NEW_WARNINGS;ERROR,Clang Warnings to fix,See Log,clang-new-warnings.log' >> ${RESULTS_DIR}/clang.txt
1219+
if $IS_DEV_BRANCH && [ $(echo "$IGNORE_BOT_TESTS" | tr ',' '\n' | grep '^CLANG-WARNINGS$' | wc -l) -eq 0 ] ; then
1220+
RUN_TESTS=false
1221+
ALL_OK=false
1222+
CLANG_BUILD_OK=false
1223+
fi
1224+
fi
1225+
1226+
if [ -d ${BUILD_LOG_DIR}/html ] ; then
1227+
mv ${BUILD_LOG_DIR}/html $WORKSPACE/clang-logs
1228+
echo 'CLANG_LOG;OK,Clang warnings summary,See Log,clang-logs' >> ${RESULTS_DIR}/clang.txt
1229+
fi
1230+
if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
1231+
echo "Errors when testing compilation with clang"
1232+
echo 'CLANG_COMPILATION_RESULTS;ERROR,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
1233+
RUN_TESTS=false
1234+
ALL_OK=false
1235+
CLANG_BUILD_OK=false
1236+
else
1237+
echo "the clang compilation had no errors/warnings!!"
1238+
echo 'CLANG_COMPILATION_RESULTS;OK,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
1239+
fi
1240+
fi
1241+
12031242
DO_PROFILING=false
12041243
DO_GPU_TESTS=false
12051244
if [ "X$BUILD_OK" = Xtrue -a "$RUN_TESTS" = "true" ]; then

0 commit comments

Comments
 (0)