From 1acb32e353ad9fab099a1768eb77152afa0be54f Mon Sep 17 00:00:00 2001 From: Marcel R Date: Wed, 20 Mar 2024 15:49:17 +0100 Subject: [PATCH 01/11] Draft for installing external aot models. --- cmssw-tool-conf.spec | 1 + pip/cms-tf-aot.file | 1 + pip/requirements.txt | 1 + py3-cms-tf-aot.spec | 18 ++++++++++++++++++ python_tools.spec | 1 + tfaot-compile.file | 34 ++++++++++++++++++++++++++++++++++ tfaot-model-test-multi.spec | 12 ++++++++++++ tfaot-model-test-simple.spec | 12 ++++++++++++ tfaot-models.file | 3 +++ 9 files changed, 83 insertions(+) create mode 100644 pip/cms-tf-aot.file create mode 100644 py3-cms-tf-aot.spec create mode 100644 tfaot-compile.file create mode 100644 tfaot-model-test-multi.spec create mode 100644 tfaot-model-test-simple.spec create mode 100644 tfaot-models.file diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 7a2d8271051..6e19182d688 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -183,6 +183,7 @@ Requires: xtl Requires: xgboost Requires: pytorch +## INCLUDE tfaot-models ## INCLUDE cmssw-vectorization ## INCLUDE cmssw-drop-tools ## INCLUDE scram-tool-conf diff --git a/pip/cms-tf-aot.file b/pip/cms-tf-aot.file new file mode 100644 index 00000000000..ef80884909e --- /dev/null +++ b/pip/cms-tf-aot.file @@ -0,0 +1 @@ +Requires: py3-PyYAML py3-cmsml diff --git a/pip/requirements.txt b/pip/requirements.txt index e13ea21652a..fa73403fcb4 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -57,6 +57,7 @@ cleo==2.0.1 click==8.1.3 clikit==0.6.2 cmsml==0.2.2 +cms-tf-aot==0.1.0 contourpy==1.0.7 correctionlib==2.2.2 crashtest==0.4.1 diff --git a/py3-cms-tf-aot.spec b/py3-cms-tf-aot.spec new file mode 100644 index 00000000000..8de3e349b0e --- /dev/null +++ b/py3-cms-tf-aot.spec @@ -0,0 +1,18 @@ +### RPM external py3-cms-tf-aot 0.1.0 +## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} + +%define github_user riga +%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff +%define branch dev +Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz + +BuildRequires: py3-pip py3-setuptools py3-wheel +Requires: py3-tensorflow py3-cmsml + +%prep +%setup -n %{n}-%{realversion} + +%build + +%install +pip3 install . --prefix=%{i} --no-deps diff --git a/python_tools.spec b/python_tools.spec index 19c9fdf261a..6002d26ee11 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -12,6 +12,7 @@ Requires: py3-keras Requires: py3-scikit-learn #save for the end Requires: py3-tensorflow +Requires: py3-cms-tf-aot Requires: py3-cmsml Requires: py3-law Requires: py3-protobuf diff --git a/tfaot-compile.file b/tfaot-compile.file new file mode 100644 index 00000000000..df2461a593f --- /dev/null +++ b/tfaot-compile.file @@ -0,0 +1,34 @@ +## tfaot common compilation and requirement file +## specs including this file should provide +## - a source archive named %{n}-%{realversion}, +## - a variable %{tool_name} that defines how tool files and include directories are named, and +## - a variable %{aot_config} that points to the aot config file of the model. + +BuildRequires: py3-cms-tf-aot +Requires: tensorflow-xla-runtime + +%prep +%setup -n %{n}-%{realversion} + +%build +cms_tf_aot_compile_model \ + --aot-config "%{aot_config}" \ + --tool-name "%{tool_name}" \ + --tool-base "%{i}" \ + --output-directory compiled_model + +%install +model_name=$( python3 -c "import yaml; print(yaml.safe_load(open('%{aot_config}'))['model']['name'])" ) + +mkdir -p %{i}/lib +mv compiled_model/*.o %{i}/lib/ + +incdir=%{i}/include/%{tool_name} +mkdir -p ${incdir} +mv compiled_model/*.h ${incdir} +pushd ${incdir} + ln -s ${model_name}.h model.h +popd + +mkdir -p %{i}/etc/scram.d +mv compiled_model/%{tool_name}.xml %{i}/etc/scram.d/ diff --git a/tfaot-model-test-multi.spec b/tfaot-model-test-multi.spec new file mode 100644 index 00000000000..c813310cd92 --- /dev/null +++ b/tfaot-model-test-multi.spec @@ -0,0 +1,12 @@ +### RPM external tfaot-model-test-multi 1.0.0 + +%define tool_name tfaot-model-test-multi + +%define github_user riga +%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff +%define branch dev +Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz + +%define aot_config test_models/multi/aot_config.yaml + +## INCLUDE tfaot-compile diff --git a/tfaot-model-test-simple.spec b/tfaot-model-test-simple.spec new file mode 100644 index 00000000000..552e129f8d8 --- /dev/null +++ b/tfaot-model-test-simple.spec @@ -0,0 +1,12 @@ +### RPM external tfaot-model-test-simple 1.0.0 + +%define tool_name tfaot-model-test-simple + +%define github_user riga +%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff +%define branch dev +Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz + +%define aot_config test_models/simple/aot_config.yaml + +## INCLUDE tfaot-compile diff --git a/tfaot-models.file b/tfaot-models.file new file mode 100644 index 00000000000..61e9cd2f4de --- /dev/null +++ b/tfaot-models.file @@ -0,0 +1,3 @@ +# test models needed by unit tests in PhysicsTools/TensorFlowAOT +Requires: tfaot-model-test-simple +Requires: tfaot-model-test-multi From 0acc62bcdb7d031f3749dade563de72fa2b4e06c Mon Sep 17 00:00:00 2001 From: Marcel R Date: Thu, 21 Mar 2024 19:43:17 +0100 Subject: [PATCH 02/11] Adapt to changes in cms_tfaot external. --- pip/requirements.txt | 2 +- py3-cms-tf-aot.spec | 18 ------------------ py3-cms-tfaot.spec | 22 ++++++++++++++++++++++ python_tools.spec | 2 +- tfaot-compile.file | 33 ++++++++++++++++----------------- tfaot-model-test-multi.spec | 9 +-------- tfaot-model-test-simple.spec | 9 +-------- 7 files changed, 42 insertions(+), 53 deletions(-) delete mode 100644 py3-cms-tf-aot.spec create mode 100644 py3-cms-tfaot.spec diff --git a/pip/requirements.txt b/pip/requirements.txt index fa73403fcb4..a642e1409d4 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -57,7 +57,7 @@ cleo==2.0.1 click==8.1.3 clikit==0.6.2 cmsml==0.2.2 -cms-tf-aot==0.1.0 +cms-tfaot==1.0.0 contourpy==1.0.7 correctionlib==2.2.2 crashtest==0.4.1 diff --git a/py3-cms-tf-aot.spec b/py3-cms-tf-aot.spec deleted file mode 100644 index 8de3e349b0e..00000000000 --- a/py3-cms-tf-aot.spec +++ /dev/null @@ -1,18 +0,0 @@ -### RPM external py3-cms-tf-aot 0.1.0 -## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} - -%define github_user riga -%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff -%define branch dev -Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz - -BuildRequires: py3-pip py3-setuptools py3-wheel -Requires: py3-tensorflow py3-cmsml - -%prep -%setup -n %{n}-%{realversion} - -%build - -%install -pip3 install . --prefix=%{i} --no-deps diff --git a/py3-cms-tfaot.spec b/py3-cms-tfaot.spec new file mode 100644 index 00000000000..aac330a6c9e --- /dev/null +++ b/py3-cms-tfaot.spec @@ -0,0 +1,22 @@ +### RPM external py3-cms-tfaot 1.0.0 +## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} + +%define github_user riga +%define tag d61fc9898bf315342873812bdd7f7c8cc59ef34f +%define branch master +Source: git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz + +BuildRequires: py3-pip py3-setuptools py3-wheel +Requires: py3-tensorflow py3-cmsml + +%prep +%setup -n %{n}-%{realversion} + +%build + +%install +pip3 install . --prefix=%{i} --no-deps + +# copy test models +mkdir -p %{i}/share +cp -r test_models %{i}/share diff --git a/python_tools.spec b/python_tools.spec index 6002d26ee11..f01fd542364 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -12,7 +12,7 @@ Requires: py3-keras Requires: py3-scikit-learn #save for the end Requires: py3-tensorflow -Requires: py3-cms-tf-aot +Requires: py3-cms-tfaot Requires: py3-cmsml Requires: py3-law Requires: py3-protobuf diff --git a/tfaot-compile.file b/tfaot-compile.file index df2461a593f..f0e5d3922fd 100644 --- a/tfaot-compile.file +++ b/tfaot-compile.file @@ -1,34 +1,33 @@ ## tfaot common compilation and requirement file -## specs including this file should provide -## - a source archive named %{n}-%{realversion}, -## - a variable %{tool_name} that defines how tool files and include directories are named, and -## - a variable %{aot_config} that points to the aot config file of the model. +## specs including this file should provide: +## 1. a variable %{aot_config}, pointing to the aot config file of the model to compile (required) +## 2. a variable %{aot_source}, referring to a fetched source to unpack during %prep (optional) +## (in this case a "Source" should be defined and %{aot_source} is likely %{n}-%{realversion}) -BuildRequires: py3-cms-tf-aot +BuildRequires: py3-cms-tfaot Requires: tensorflow-xla-runtime %prep -%setup -n %{n}-%{realversion} +%if "%{?aot_source}" +%setup -n %{aot_source} +%endif %build -cms_tf_aot_compile_model \ +cms_tfaot_compile \ --aot-config "%{aot_config}" \ - --tool-name "%{tool_name}" \ + --tool-name "%{n}" \ --tool-base "%{i}" \ --output-directory compiled_model %install -model_name=$( python3 -c "import yaml; print(yaml.safe_load(open('%{aot_config}'))['model']['name'])" ) - mkdir -p %{i}/lib mv compiled_model/*.o %{i}/lib/ -incdir=%{i}/include/%{tool_name} -mkdir -p ${incdir} -mv compiled_model/*.h ${incdir} -pushd ${incdir} - ln -s ${model_name}.h model.h -popd +mkdir -p %{i}/include/%{n} +mv compiled_model/*.h %{i}/include/%{n} mkdir -p %{i}/etc/scram.d -mv compiled_model/%{tool_name}.xml %{i}/etc/scram.d/ +mv compiled_model/%{n}.xml %{i}/etc/scram.d/ + +%post +%{relocateConfig}etc/scram.d/%{n}.xml diff --git a/tfaot-model-test-multi.spec b/tfaot-model-test-multi.spec index c813310cd92..0def7f899d0 100644 --- a/tfaot-model-test-multi.spec +++ b/tfaot-model-test-multi.spec @@ -1,12 +1,5 @@ ### RPM external tfaot-model-test-multi 1.0.0 -%define tool_name tfaot-model-test-multi - -%define github_user riga -%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff -%define branch dev -Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz - -%define aot_config test_models/multi/aot_config.yaml +%define aot_config $PY3_CMS_TFAOT_ROOT/share/test_models/multi/aot_config.yaml ## INCLUDE tfaot-compile diff --git a/tfaot-model-test-simple.spec b/tfaot-model-test-simple.spec index 552e129f8d8..013b51f69aa 100644 --- a/tfaot-model-test-simple.spec +++ b/tfaot-model-test-simple.spec @@ -1,12 +1,5 @@ ### RPM external tfaot-model-test-simple 1.0.0 -%define tool_name tfaot-model-test-simple - -%define github_user riga -%define tag ac5e4ed8507ad63be5814247ef32cb9d0ecc21ff -%define branch dev -Source: git+https://github.com/%{github_user}/cms-tf-aot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz - -%define aot_config test_models/simple/aot_config.yaml +%define aot_config $PY3_CMS_TFAOT_ROOT/share/test_models/simple/aot_config.yaml ## INCLUDE tfaot-compile From 791f5b9ee4fab418c72a3bb06fc8ca64c0efa0f9 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Thu, 21 Mar 2024 19:49:50 +0100 Subject: [PATCH 03/11] Typo. --- pip/{cms-tf-aot.file => cms-tfaot.file} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pip/{cms-tf-aot.file => cms-tfaot.file} (100%) diff --git a/pip/cms-tf-aot.file b/pip/cms-tfaot.file similarity index 100% rename from pip/cms-tf-aot.file rename to pip/cms-tfaot.file From bca654e6a2c881a4a841c555e0735b979b9cbcc9 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Thu, 21 Mar 2024 20:32:58 +0100 Subject: [PATCH 04/11] Update cms-tfaot. --- py3-cms-tfaot.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-cms-tfaot.spec b/py3-cms-tfaot.spec index aac330a6c9e..e52de50a738 100644 --- a/py3-cms-tfaot.spec +++ b/py3-cms-tfaot.spec @@ -2,7 +2,7 @@ ## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} %define github_user riga -%define tag d61fc9898bf315342873812bdd7f7c8cc59ef34f +%define tag b5b7bb48a1dfe0ddee9277276bbba7f6af826c62 %define branch master Source: git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz From 9c78bfb4d3cd19602c0bcf40b9905636b32632ba Mon Sep 17 00:00:00 2001 From: Marcel R Date: Fri, 22 Mar 2024 11:29:28 +0100 Subject: [PATCH 05/11] Update cms-tfaot. --- py3-cms-tfaot.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-cms-tfaot.spec b/py3-cms-tfaot.spec index e52de50a738..3ebb5dd5dc4 100644 --- a/py3-cms-tfaot.spec +++ b/py3-cms-tfaot.spec @@ -2,7 +2,7 @@ ## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} %define github_user riga -%define tag b5b7bb48a1dfe0ddee9277276bbba7f6af826c62 +%define tag a2bbd06cbed0efa1fa191cc2f50a6b03067e59d2 %define branch master Source: git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz From 1ff0e6e362e88c0e0ba9ad1e2f6d80c7b159f51e Mon Sep 17 00:00:00 2001 From: Marcel R Date: Fri, 22 Mar 2024 17:16:03 +0100 Subject: [PATCH 06/11] Build shared xla runtime library. --- py3-cms-tfaot.spec | 2 +- .../tensorflow-xla-runtime/tensorflow-xla-runtime.xml | 6 ++++-- tensorflow-xla-runtime.spec | 9 ++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/py3-cms-tfaot.spec b/py3-cms-tfaot.spec index 3ebb5dd5dc4..91c53d7143c 100644 --- a/py3-cms-tfaot.spec +++ b/py3-cms-tfaot.spec @@ -7,7 +7,7 @@ Source: git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz BuildRequires: py3-pip py3-setuptools py3-wheel -Requires: py3-tensorflow py3-cmsml +Requires: py3-cmsml py3-tensorflow %prep %setup -n %{n}-%{realversion} diff --git a/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml b/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml index c022bc11694..ca531c2d2d9 100644 --- a/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml +++ b/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml @@ -1,9 +1,11 @@ - + - + + + diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec index 44ad2cf9ade..6e67c560c44 100644 --- a/tensorflow-xla-runtime.spec +++ b/tensorflow-xla-runtime.spec @@ -4,7 +4,7 @@ Source99: scram-tools.file/tools/eigen/env -Requires: eigen py3-tensorflow +Requires: eigen py3-tensorflow abseil-cpp BuildRequires: cmake %prep @@ -12,6 +12,7 @@ BuildRequires: cmake cp -r ${PY3_TENSORFLOW_ROOT}/lib/python%{cms_python3_major_minor_version}/site-packages/tensorflow . %build + source %{_sourcedir}/env export CPATH="${CPATH}:${EIGEN_ROOT}/include/eigen3" @@ -21,11 +22,17 @@ CXXFLAGS="-fPIC %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}" %endif pushd tensorflow/xla_aot_runtime_src + # the CMakelists.txt file is not intended to produce a shared library so create a static one first cmake . -DCMAKE_CXX_FLAGS="${CXXFLAGS}" -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} -DBUILD_SHARED_LIBS=OFF make %{makeprocesses} + + # build the shared library, linking missing symbols from abseil + gcc -shared -o libtf_xla_runtime.so -Wl,--whole-archive libtf_xla_runtime.a -Wl,--no-whole-archive \ + -L${ABSEIL_CPP_ROOT}/lib -labsl_strings -labsl_str_format_internal popd %install mkdir -p %{i}/lib/archive mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.a %{i}/lib/archive/libtf_xla_runtime-static.a +mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.so %{i}/lib/ From 78384488992b22d118a8be19009b630745fa6a1d Mon Sep 17 00:00:00 2001 From: Marcel R Date: Mon, 25 Mar 2024 18:55:25 +0100 Subject: [PATCH 07/11] Address review comments by @smuzaffar. --- pip/cms-tfaot.file | 11 +++++++++++ py3-cms-tfaot.spec | 22 ---------------------- tensorflow-xla-runtime-absl.patch | 12 ++++++++++++ tensorflow-xla-runtime.spec | 19 ++++++++++--------- 4 files changed, 33 insertions(+), 31 deletions(-) delete mode 100644 py3-cms-tfaot.spec create mode 100644 tensorflow-xla-runtime-absl.patch diff --git a/pip/cms-tfaot.file b/pip/cms-tfaot.file index ef80884909e..706612e84a7 100644 --- a/pip/cms-tfaot.file +++ b/pip/cms-tfaot.file @@ -1 +1,12 @@ +BuildRequires: py3-pip py3-setuptools py3-wheel Requires: py3-PyYAML py3-cmsml + +%define github_user riga +%define tag a2bbd06cbed0efa1fa191cc2f50a6b03067e59d2 +%define branch master +%define source0 git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz + +# copy test models +%define PipPostInstall \ + mkdir -p %{i}/share; \ + cp -r cmsdist-tmp/pip-req-build-*/test_models %{i}/share/ diff --git a/py3-cms-tfaot.spec b/py3-cms-tfaot.spec deleted file mode 100644 index 91c53d7143c..00000000000 --- a/py3-cms-tfaot.spec +++ /dev/null @@ -1,22 +0,0 @@ -### RPM external py3-cms-tfaot 1.0.0 -## INITENV +PATH PYTHON3PATH %i/${PYTHON3_LIB_SITE_PACKAGES} - -%define github_user riga -%define tag a2bbd06cbed0efa1fa191cc2f50a6b03067e59d2 -%define branch master -Source: git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz - -BuildRequires: py3-pip py3-setuptools py3-wheel -Requires: py3-cmsml py3-tensorflow - -%prep -%setup -n %{n}-%{realversion} - -%build - -%install -pip3 install . --prefix=%{i} --no-deps - -# copy test models -mkdir -p %{i}/share -cp -r test_models %{i}/share diff --git a/tensorflow-xla-runtime-absl.patch b/tensorflow-xla-runtime-absl.patch new file mode 100644 index 00000000000..afc9ec1a1af --- /dev/null +++ b/tensorflow-xla-runtime-absl.patch @@ -0,0 +1,12 @@ +--- tensorflow/xla_aot_runtime_src/CMakeLists.txt 2024-03-24 08:28:34.000000000 +0100 ++++ tensorflow/xla_aot_runtime_src/CMakeLists.txt 2024-03-25 11:17:58.108587945 +0100 +@@ -14,6 +14,8 @@ + -Wno-sign-compare + ) + +-add_library(tf_xla_runtime STATIC ++find_package(absl REQUIRED) ++add_library(tf_xla_runtime SHARED + $ + ) ++target_link_libraries(tf_xla_runtime absl::strings absl::str_format_internal) diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec index 6e67c560c44..af3cf0c854d 100644 --- a/tensorflow-xla-runtime.spec +++ b/tensorflow-xla-runtime.spec @@ -4,12 +4,15 @@ Source99: scram-tools.file/tools/eigen/env +Patch0: tensorflow-xla-runtime-absl + Requires: eigen py3-tensorflow abseil-cpp BuildRequires: cmake %prep cp -r ${PY3_TENSORFLOW_ROOT}/lib/python%{cms_python3_major_minor_version}/site-packages/tensorflow . +%patch -p0 %build @@ -18,21 +21,19 @@ export CPATH="${CPATH}:${EIGEN_ROOT}/include/eigen3" CXXFLAGS="-fPIC %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}" %ifarch x86_64 - CXXFLAGS="${CXXFLAGS} -msse3" + CXXFLAGS="${CXXFLAGS} -msse3" %endif pushd tensorflow/xla_aot_runtime_src - # the CMakelists.txt file is not intended to produce a shared library so create a static one first - cmake . -DCMAKE_CXX_FLAGS="${CXXFLAGS}" -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} -DBUILD_SHARED_LIBS=OFF + cmake . \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} \ + -DCMAKE_PREFIX_PATH=${ABSEIL_CPP_ROOT} \ + -DBUILD_SHARED_LIBS=ON make %{makeprocesses} - - # build the shared library, linking missing symbols from abseil - gcc -shared -o libtf_xla_runtime.so -Wl,--whole-archive libtf_xla_runtime.a -Wl,--no-whole-archive \ - -L${ABSEIL_CPP_ROOT}/lib -labsl_strings -labsl_str_format_internal popd %install -mkdir -p %{i}/lib/archive -mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.a %{i}/lib/archive/libtf_xla_runtime-static.a +mkdir -p %{i}/lib mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.so %{i}/lib/ From d2d25757c59574d60f74fea866098ebcca529420 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Tue, 26 Mar 2024 16:08:07 +0100 Subject: [PATCH 08/11] Remove unnecessary units using missing symbols. --- .../tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml | 1 - tensorflow-xla-runtime.spec | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml b/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml index ca531c2d2d9..c51e6e12bf0 100644 --- a/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml +++ b/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml @@ -4,7 +4,6 @@ - diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec index af3cf0c854d..00155a86576 100644 --- a/tensorflow-xla-runtime.spec +++ b/tensorflow-xla-runtime.spec @@ -19,12 +19,15 @@ cp -r ${PY3_TENSORFLOW_ROOT}/lib/python%{cms_python3_major_minor_version}/site-p source %{_sourcedir}/env export CPATH="${CPATH}:${EIGEN_ROOT}/include/eigen3" -CXXFLAGS="-fPIC %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}" +CXXFLAGS="-fPIC -Wl,-z,defs %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}" %ifarch x86_64 CXXFLAGS="${CXXFLAGS} -msse3" %endif pushd tensorflow/xla_aot_runtime_src + # remove unnecessary implementations that use symbols that are not even existing + rm tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc + cmake . \ -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} \ From 0d650f69cc45c4991691f3b00902c607bc2399a1 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Wed, 27 Mar 2024 10:34:49 +0100 Subject: [PATCH 09/11] Update tf python tools, run relocation on headers. --- pip/cms-tfaot.file | 2 +- pip/requirements.txt | 2 +- tfaot-compile.file | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pip/cms-tfaot.file b/pip/cms-tfaot.file index 706612e84a7..52ec93996f7 100644 --- a/pip/cms-tfaot.file +++ b/pip/cms-tfaot.file @@ -2,7 +2,7 @@ BuildRequires: py3-pip py3-setuptools py3-wheel Requires: py3-PyYAML py3-cmsml %define github_user riga -%define tag a2bbd06cbed0efa1fa191cc2f50a6b03067e59d2 +%define tag 67ff5162a6030aad4f23aaacb67ea4f7f28109e1 %define branch master %define source0 git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz diff --git a/pip/requirements.txt b/pip/requirements.txt index a642e1409d4..478324ae59b 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -56,7 +56,7 @@ charset-normalizer==3.1.0 cleo==2.0.1 click==8.1.3 clikit==0.6.2 -cmsml==0.2.2 +cmsml==0.2.4 cms-tfaot==1.0.0 contourpy==1.0.7 correctionlib==2.2.2 diff --git a/tfaot-compile.file b/tfaot-compile.file index f0e5d3922fd..9cb766fb30d 100644 --- a/tfaot-compile.file +++ b/tfaot-compile.file @@ -31,3 +31,4 @@ mv compiled_model/%{n}.xml %{i}/etc/scram.d/ %post %{relocateConfig}etc/scram.d/%{n}.xml +%relocateConfigAll include/%{n} *.h From 0677c1b66b90326b47d191b54109f4c38d1af3c3 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Wed, 27 Mar 2024 16:29:57 +0100 Subject: [PATCH 10/11] Forward build arch via --target_triple to aot compilation. --- pip/cms-tfaot.file | 2 +- pip/requirements.txt | 2 +- tfaot-compile.file | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pip/cms-tfaot.file b/pip/cms-tfaot.file index 52ec93996f7..dea73a5afe4 100644 --- a/pip/cms-tfaot.file +++ b/pip/cms-tfaot.file @@ -2,7 +2,7 @@ BuildRequires: py3-pip py3-setuptools py3-wheel Requires: py3-PyYAML py3-cmsml %define github_user riga -%define tag 67ff5162a6030aad4f23aaacb67ea4f7f28109e1 +%define tag a9ee8f7d7091015a496f4be0880763a23b5287ab %define branch master %define source0 git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz diff --git a/pip/requirements.txt b/pip/requirements.txt index 478324ae59b..d7bddb3e308 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -56,7 +56,7 @@ charset-normalizer==3.1.0 cleo==2.0.1 click==8.1.3 clikit==0.6.2 -cmsml==0.2.4 +cmsml==0.2.5 cms-tfaot==1.0.0 contourpy==1.0.7 correctionlib==2.2.2 diff --git a/tfaot-compile.file b/tfaot-compile.file index 9cb766fb30d..c5952a66420 100644 --- a/tfaot-compile.file +++ b/tfaot-compile.file @@ -7,6 +7,12 @@ BuildRequires: py3-cms-tfaot Requires: tensorflow-xla-runtime +%ifarch ppc64le +%define build_arch powerpc64le-unknown-linux-gnu +%else +%define build_arch %{_arch}-unknown-linux-gnu +%endif + %prep %if "%{?aot_source}" %setup -n %{aot_source} @@ -17,7 +23,8 @@ cms_tfaot_compile \ --aot-config "%{aot_config}" \ --tool-name "%{n}" \ --tool-base "%{i}" \ - --output-directory compiled_model + --output-directory compiled_model \ + --additional-flags="--target_triple %{build_arch}" %install mkdir -p %{i}/lib From a964e74035ef2f14cd5cab61516dcc6b97fb3535 Mon Sep 17 00:00:00 2001 From: Marcel R Date: Thu, 28 Mar 2024 11:50:47 +0100 Subject: [PATCH 11/11] Use newest cms-tfaot from externals. --- pip/cms-tfaot.file | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip/cms-tfaot.file b/pip/cms-tfaot.file index dea73a5afe4..45e1a3c66d2 100644 --- a/pip/cms-tfaot.file +++ b/pip/cms-tfaot.file @@ -1,8 +1,8 @@ BuildRequires: py3-pip py3-setuptools py3-wheel Requires: py3-PyYAML py3-cmsml -%define github_user riga -%define tag a9ee8f7d7091015a496f4be0880763a23b5287ab +%define github_user cms-externals +%define tag a03ccef81c610e051d3d7d172d675fe6c1ff17e2 %define branch master %define source0 git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz