Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
196 changes: 14 additions & 182 deletions cmssw-tool-conf.spec
Original file line number Diff line number Diff line change
Expand Up @@ -5,190 +5,22 @@
## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools
## UPLOAD_DEPENDENCIES dqmgui

Requires: crab
Requires: cmssw-wm-tools
Requires: google-benchmark-toolfile
Requires: catch2-toolfile
Requires: starlight-toolfile
Requires: alpgen-toolfile
Requires: boost-toolfile
Requires: bz2lib-toolfile
Requires: charybdis-toolfile
Requires: classlib-toolfile
Requires: clhep-toolfile
Requires: coral-toolfile
Requires: cppunit-toolfile
Requires: curl-toolfile
Requires: das_client-toolfile
Requires: db6-toolfile
Requires: davix-toolfile
Requires: evtgen-toolfile
Requires: expat-toolfile
Requires: fakesystem
Requires: fastjet-toolfile
Requires: flatbuffers-toolfile
Requires: fmt-toolfile
Requires: gbl-toolfile
Requires: gcc-toolfile
Requires: gdbm-toolfile
Requires: geant4-toolfile
Requires: geant4data-toolfile
Requires: vecgeom-toolfile
Requires: glimpse-toolfile
Requires: gmake-toolfile
Requires: gsl-toolfile
Requires: highfive-toolfile
Requires: hector-toolfile
Requires: hepmc-toolfile
Requires: heppdt-toolfile
Requires: herwig-toolfile
Requires: herwig7-toolfile
Requires: hydjet-toolfile
Requires: ittnotify-toolfile
Requires: jemalloc-toolfile
Requires: jemalloc-debug-toolfile
Requires: jimmy-toolfile
Requires: json-toolfile
Requires: ktjet-toolfile
Requires: lhapdf-toolfile
Requires: libjpeg-turbo-toolfile
Requires: libpng-toolfile
Requires: libtiff-toolfile
Requires: libungif-toolfile
Requires: libxml2-toolfile
Requires: lwtnn-toolfile
Requires: meschach-toolfile
Requires: pcre-toolfile
Requires: photos-toolfile
Requires: photospp-toolfile
Requires: pyquen-toolfile
Requires: pythia6-toolfile
Requires: pythia8-toolfile
Requires: python-toolfile
Requires: python3-toolfile
Requires: root-toolfile
Requires: sherpa-toolfile
Requires: libpciaccess-toolfile
Requires: numactl-toolfile
Requires: hwloc-toolfile
%ifnarch aarch64
Requires: gdrcopy-toolfile
%endif
Requires: ucx-toolfile
Requires: openmpi-toolfile
Requires: sigcpp-toolfile
Requires: sqlite-toolfile
Requires: systemtools
Requires: tauola-toolfile
Requires: tauolapp-toolfile
Requires: thepeg-toolfile
Requires: toprex-toolfile
Requires: libuuid-toolfile
Requires: xerces-c-toolfile
Requires: zlib-toolfile
Requires: dcap-toolfile
Requires: frontier_client-toolfile
Requires: xrootd-toolfile
Requires: dd4hep-toolfile
Requires: graphviz-toolfile
Requires: valgrind-toolfile
Requires: cmsswdata-toolfile
Requires: zstd-toolfile
Requires: hls-toolfile
Requires: opencv-toolfile
Requires: grpc-toolfile
Requires: onnxruntime-toolfile
Requires: triton-inference-server-toolfile
Requires: hdf5-toolfile
Requires: rivet-toolfile
Requires: cascade-toolfile
Requires: yoda-toolfile
Requires: fftw3-toolfile
Requires: fftjet-toolfile
Requires: professor2-toolfile
Requires: xz-toolfile
Requires: lz4-toolfile
Requires: protobuf-toolfile
Requires: lcov-toolfile
Requires: llvm-gcc-toolfile
Requires: tbb-toolfile
Requires: mctester-toolfile
Requires: vdt-toolfile
Requires: icc-gcc-toolfile
Requires: gnuplot-toolfile
Requires: sloccount-toolfile
Requires: millepede-toolfile
Requires: pacparser-toolfile
Requires: git-toolfile
Requires: cgal-toolfile
Requires: doxygen-toolfile
Requires: yaml-cpp-toolfile
Requires: gmp-static-toolfile
Requires: mpfr-static-toolfile
Requires: fastjet-contrib-toolfile
Requires: opencl-toolfile
Requires: opencl-cpp-toolfile
Requires: qd-toolfile
Requires: blackhat-toolfile
Requires: sherpa-toolfile
Requires: geant4-parfullcms-toolfile
Requires: fasthadd
Requires: eigen-toolfile
Requires: gdb-toolfile
Requires: libxslt-toolfile
Requires: giflib-toolfile
Requires: freetype-toolfile
Requires: utm-toolfile
Requires: libffi-toolfile
Requires: CSCTrackFinderEmulation-toolfile
Requires: tinyxml2-toolfile
Requires: md5-toolfile
Requires: gosamcontrib-toolfile
Requires: gosam-toolfile
Requires: madgraph5amcatnlo-toolfile
Requires: python_tools
Requires: dasgoclient
Requires: OpenBLAS-toolfile
Requires: mxnet-predict-toolfile
Requires: mkfit-toolfile
Requires: dablooms-toolfile

# Only for Linux platform.
%ifos linux
Requires: openldap-toolfile
Requires: gperftools-toolfile
Requires: cuda-toolfile
Requires: alpaka-toolfile
Requires: cupla-toolfile

%if "%{cmsos}" != "slc7_aarch64"
Requires: cudnn-toolfile
%endif

Requires: libunwind-toolfile
%ifnarch ppc64le
Requires: igprof-toolfile
Requires: openloops-toolfile
%endif

%ifarch x86_64
Requires: dmtcp-toolfile
Requires: tkonlinesw-toolfile
Requires: oracle-toolfile
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mrodozov , we need to keep the dependencies but should replace <tool>-toolfile with <tool> only.
This should also be done for coral and fwlite tool-conf.spec

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this file is containing a short list because so I first finish pkgs with any script logic, the short list if for not deploying all xmls every time (not to build the whole cmssw-tool-conf)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this type of changes are better to test locally as this will help with debugging.

BuildRequires: scram-tools
Requires: gcc
Requires: herwig
Requires: libpng
Requires: OpenBLAS
Requires: root
Requires: tbb
Requires: zlib
Requires: zstd
Requires: intel-vtune
Requires: cmsmon-tools
Requires: dip-toolfile
%else
Requires: tkonlinesw-fake-toolfile
Requires: oracle-fake-toolfile
%endif
%endif

Requires: tensorflow-toolfile
Requires: xtensor-toolfile
Requires: xtl-toolfile
Requires: xgboost-toolfile
Requires: das_client
Requires: llvm
Requires: boost

%define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug

# the vectorization flags/macroses will be available

## IMPORT scramv1-tool-conf
16 changes: 16 additions & 0 deletions scram-tools.spec
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
### RPM cms scram-tools 1.0

%define branch main
%define tag 33e20671985a2b7774681adcdfce655c0186b650
%define github_user cms-sw
%define github_repo %{n}

Source: git+https://github.com/%{github_user}/%{github_repo}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz

%prep
%setup -n %{n}-%{realversion}

%build

%install
cp -r %{_builddir}/%{n}-%{realversion} %{i}/%{n}
68 changes: 50 additions & 18 deletions scramv1-tool-conf.file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,22 @@ Source: none
%define CheckScramTools \
uctool=`echo $tool | tr '[a-z-]' '[A-Z_]'` \
toolbase=`eval echo \\\\$${uctool}_ROOT` \
[ -d $toolbase/etc/scram.d ] || continue
[ -d ${SCRAM_TOOLS_ROOT}/scram-tools/$tool ] || continue

%define CopyScramTools \
echo ">> Copying tool files from: $tool" \
for xml in $(find $toolbase/etc/scram.d -type f) ; do \
for xml in $(find ${SCRAM_TOOLS_ROOT}/scram-tools/$tool -type f -name "*.xml") ; do \
bxml=$(basename $xml | tr '[A-Z]' '[a-z]') \
[ -f %i/tools/selected/$bxml ] && continue \
[ -f %i/tools/available/$bxml ] && continue \
cp $xml %i/tools/selected/${bxml} \
T_VERSION=`eval echo \\\\$${uctool}_VERSION` \
sed -i -e 's|@TOOL_ROOT@|'"$toolbase"'|g' %i/tools/selected/${bxml} \
sed -i -e 's|@TOOL_VERSION@|'"$T_VERSION"'|g' %i/tools/selected/${bxml} \
echo " Copied $bxml" \
done

done \
tool_script=${SCRAM_TOOLS_ROOT}/scram-tools/$tool/$(echo $tool | tr '[A-Z]' '[a-z]').sh \
if [ -f $tool_script ]; then ($tool_script %i/tools/selected); fi
Copy link
Contributor

@smuzaffar smuzaffar Jul 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mrodozov , I was thinking to move all this CopyScramTools logic in a script which we can call like ${SCRAM_TOOLS_ROOT}/get_tools <tool_name> <outdir>. This tool then can process the <toolname>/<toolname>*.xml files and replace @TOOL_ROOT@ and @TOOL_VERSION@ and can internally call ${SCRAM_TOOLS_ROOT}/get_tools/<toolname>/<toolname>.sh if available

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll move that part in a file to be called as a script with args then

Copy link
Contributor

@smuzaffar smuzaffar Jul 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the ${SCRAM_TOOLS_ROOT}/get_tools script can take care of common variables e.g.

TOOL_ROOT
TOOL_VERSION
PYTHONV
PYTHON3V

and rest can go in tool's specific script. The tool's specific script should only set the environment variables e.g.

COMPILER_NAME_SUFFIX
OS_CSHAREDOBJECTFLAGS
ARCH_CSHAREDOBJECTFLAGS
BOOST_CHRONO_LIB
BOOST_DATE_TIME_LIB

and the get_tools script should do the replacement. e.g. get_tools can do source gcc/gcc.sh ( which should set the env) and then get_tools should do the replacement of @VARIABLE@

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mrodozov , please take note of my comment #7110 (comment) too

%prep
%build

Expand All @@ -42,6 +46,23 @@ mkdir -p %i/tools/selected %i/tools/available
%define skipreqtools %{nil}
%endif

%if "%{?package_vectorization}" != ""
PKG_VECTORIZATION=`echo %package_vectorization`
export PKG_VECTORIZATION
%endif

COMP_ARCH_SPECIFIC_FLAGS=""
%ifarch x86_64
COMP_ARCH_SPECIFIC_FLAGS=$(for v in %package_vectorization ; do %{cmsdist_directory}/vectorization/cmsdist_packages.py ${v} ; done)
%endif
%ifarch aarch64
COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields"
%endif
%ifarch ppc64le
COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields %ppc64le_build_flags"
%endif
export COMP_ARCH_SPECIFIC_FLAGS

DoneTools=" "
SkipTools=
for tool in %requiredtools; do
Expand Down Expand Up @@ -69,23 +90,11 @@ for tool in $SkipTools ; do
%{CopyScramTools}
done

# Fixes logic in above loop in case of online release:
# the descriptions of explicitly defined systemtools should
# take precedence over those from cms-rebuilt rpms
%if "%online" == "true"
for tool in %systemtools ${PKGTOOLS_SYSTEM_TOOLS} ; do
lc_tool=$(echo $tool | tr '[A-Z]' '[a-z]')
rm -f %i/tools/selected/${lc_tool} %i/tools/selected/${lc_tool}.xml
cp -rf %{onlinesystemtoolsroot}/etc/scram.d/${tool}.xml %i/tools/selected/${lc_tool}.xml
done
# For now copy all systemtools files. If needed, this can be done more selectively.
%endif

for stool in $(echo %skipreqtools | tr '[A-Z]' '[a-z]') ; do
[ -f %i/tools/selected/${stool}.xml ] || continue
mv %i/tools/selected/${stool}.xml %i/tools/available
done

if [ -e $SCRAMV1_ROOT/bin/chktool ] ; then
find %i/tools -name '*.xml' -type f | (xargs $SCRAMV1_ROOT/bin/chktool > %i/errors.log 2>&1 || true)
ToolErrors=`cat %i/errors.log | wc -l | awk '{print $1}'`
Expand Down Expand Up @@ -141,6 +150,10 @@ for pkg in $(echo %{allpkgreqs} | tr ' ' '\n' | grep -v '\-toolfile/' | grep '/
echo "</tool>" >> %{i}/tools/selected/${pk_name}.xml
done
DUP_BIN=$(echo "${ALL_PY_BIN}" | tr ' ' '\n' | sort | uniq -c | sed 's|^\s*||' | grep -v '^1 ' | sed 's|^.* ||')

echo "echoing the install path"
echo %i

set +x
if [ "${DUP_BIN}" != "" ] ; then
for p in ${DUP_BIN} ; do
Expand All @@ -150,7 +163,26 @@ if [ "${DUP_BIN}" != "" ] ; then
exit 1
fi
set -x

#vectorized packages
%if "%{?vectorized_packages}" != ""
for vect_package in %vectorized_packages; do
echo $vect_package
package_uc=$(echo $vect_package | tr '[a-z-]' '[A-Z_]')
r=""
#echo $package_uc
for v in $(echo %{package_vectorization} | tr '[a-z-]' '[A-Z_]'); do
r=$r" <environment name=\"${v}_LIBDIR\" default=\""
r=$r'\$'
r=$r${package_uc}_${v}_ROOT/lib"\" type=\"path\"/>\n"
#echo $v
done
r=$r'</client>'
perl -p -i -e "s|</client>|${r}|g" %i/tools/selected/$(echo $vect_package | tr '[A-Z]' '[a-z]').xml
done
%endif

%post

%{relocateCmsFiles} $(find $RPM_INSTALL_PREFIX/%{pkgrel}/tools -type f)
%{relocateCmsFiles} $(find $RPM_INSTALL_PREFIX/%{pkgrel}/lib -type f)