From 9d12c58b67411875af559dfc5108a2d286a95d3c Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 2 Jul 2024 09:21:39 -0600 Subject: [PATCH 01/69] Add gpmdk for the Venado hackathon --- .gitmodules | 4 + bml | 1 + examples/gpmdk/README.rst | 97 + examples/gpmdk/build_chicoma_hackathon.sh | 10 + examples/gpmdk/docs/Makefile | 19 + .../gpmdk/docs/source-doxygen/gpmd_doxyfile | 407 ++++ examples/gpmdk/docs/source-sphinx/Makefile | 21 + .../gpmdk/docs/source-sphinx/TUTORIAL.rst | 13 + examples/gpmdk/docs/source-sphinx/conf.py | 28 + examples/gpmdk/docs/source-sphinx/index.rst | 22 + examples/gpmdk/run/coords_2088.dat | 2092 +++++++++++++++++ examples/gpmdk/run/coords_300.dat | 304 +++ examples/gpmdk/run/get_energy.py | 17 + examples/gpmdk/run/gpmd_input.in | 153 ++ examples/gpmdk/run/input.in | 182 ++ examples/gpmdk/run/job | 19 + examples/gpmdk/run/latte.in | 74 + examples/gpmdk/run/run.sh | 14 + examples/gpmdk/run/sbatch.sh | 20 + examples/gpmdk/run/wrapper.sh | 10 + examples/gpmdk/setenv_chicoma_hackathon.sh | 6 + examples/gpmdk/src/CMakeLists.txt | 209 ++ examples/gpmdk/src/gpmd.py | 122 + examples/gpmdk/src/gpmd_secuential.F90 | 69 + examples/gpmdk/src/gpmdcov_allocation.F90 | 143 ++ examples/gpmdk/src/gpmdcov_assert.F90 | 54 + examples/gpmdk/src/gpmdcov_buildz.F90 | 25 + examples/gpmdk/src/gpmdcov_constraints.F90 | 147 ++ examples/gpmdk/src/gpmdcov_diagonalize.F90 | 158 ++ examples/gpmdk/src/gpmdcov_dm_min.F90 | 600 +++++ examples/gpmdk/src/gpmdcov_dump.F90 | 38 + examples/gpmdk/src/gpmdcov_energandforces.F90 | 379 +++ examples/gpmdk/src/gpmdcov_field.F90 | 48 + examples/gpmdk/src/gpmdcov_finalize.F90 | 195 ++ examples/gpmdk/src/gpmdcov_firstcharges.F90 | 131 ++ examples/gpmdk/src/gpmdcov_graphpart.F90 | 140 ++ examples/gpmdk/src/gpmdcov_highorder.F90 | 248 ++ examples/gpmdk/src/gpmdcov_init.F90 | 243 ++ examples/gpmdk/src/gpmdcov_initparts.F90 | 116 + examples/gpmdk/src/gpmdcov_kernel.F90 | 1272 ++++++++++ examples/gpmdk/src/gpmdcov_langevin.F90 | 261 ++ examples/gpmdk/src/gpmdcov_latte.F90 | 149 ++ examples/gpmdk/src/gpmdcov_lib.F90 | 138 ++ examples/gpmdk/src/gpmdcov_lib_mod.F90 | 219 ++ examples/gpmdk/src/gpmdcov_mdloop.F90 | 532 +++++ examples/gpmdk/src/gpmdcov_mod.F90 | 409 ++++ examples/gpmdk/src/gpmdcov_mod_orig.F90 | 91 + examples/gpmdk/src/gpmdcov_neighbor.F90 | 1445 ++++++++++++ examples/gpmdk/src/gpmdcov_nonequilibrium.F90 | 114 + examples/gpmdk/src/gpmdcov_parser.F90 | 299 +++ examples/gpmdk/src/gpmdcov_part.F90 | 280 +++ examples/gpmdk/src/gpmdcov_preparemd.F90 | 95 + examples/gpmdk/src/gpmdcov_rankN.F90 | 38 + examples/gpmdk/src/gpmdcov_reshuffle.F90 | 73 + examples/gpmdk/src/gpmdcov_restart.F90 | 57 + examples/gpmdk/src/gpmdcov_rhosolver.F90 | 44 + examples/gpmdk/src/gpmdcov_test.F90 | 14 + examples/gpmdk/src/gpmdcov_vars.F90 | 135 ++ examples/gpmdk/src/gpmdcov_writeout.F90 | 245 ++ examples/gpmdk/src/gpmdk.F90 | 120 + examples/gpmdk/tests/get_energy.py | 17 + examples/gpmdk/tests/input_BlockPart.in | 55 + examples/gpmdk/tests/input_DiagEf.in | 56 + .../tests/input_KerByPartsBuildAlwaysMetis.in | 84 + .../tests/input_KerByPartsBuildOnceBlock.in | 84 + .../input_KerByPartsBuildOnceBlockUpdate.in | 85 + .../input_KerByPartsBuildOnceMetisUpdate.in | 85 + .../gpmdk/tests/input_KerFullBuildAlways.in | 84 + .../tests/input_KerFullBuildAlwaysBlock.in | 84 + .../tests/input_KerFullBuildAlwaysMetis.in | 84 + .../gpmdk/tests/input_KerFullBuildOnce.in | 84 + .../tests/input_KerFullBuildOnceBlock.in | 84 + .../tests/input_KerFullBuildOnceMetis.in | 84 + examples/gpmdk/tests/input_KerUpdate.in | 76 + examples/gpmdk/tests/input_test.in | 57 + .../tests/latteTBparams/bondints.nonortho | 42 + .../latteTBparams/bondints.nonortho.algo | 41 + .../gpmdk/tests/latteTBparams/bondints.ortho | 53 + .../tests/latteTBparams/electrons.dat.algo | 7 + .../gpmdk/tests/latteTBparams/ppots.nonortho | 13 + .../tests/latteTBparams/ppots.nonortho.algo | 12 + .../gpmdk/tests/latteTBparams/ppots.ortho | 15 + examples/gpmdk/tests/ref.BlockPart | 10 + examples/gpmdk/tests/ref.DiagEf | 10 + examples/gpmdk/tests/ref.KerUpdate | 101 + examples/gpmdk/tests/ref.gpmdcov_DM_Min | 1705 ++++++++++++++ examples/gpmdk/tests/ref.gpmdcov_FirstCharges | 362 +++ examples/gpmdk/tests/ref.gpmdcov_Init | 422 ++++ examples/gpmdk/tests/ref.gpmdcov_InitParts | 28 + examples/gpmdk/tests/ref.gpmdcov_MDloop | 422 ++++ examples/gpmdk/tests/ref.gpmdcov_Part | 28 + examples/gpmdk/tests/ref.gpmdcov_PrepareMD | 422 ++++ examples/gpmdk/tests/run_test.sh | 68 + examples/gpmdk/tests/test-energy.py | 56 + 94 files changed, 17528 insertions(+) create mode 100644 .gitmodules create mode 160000 bml create mode 100644 examples/gpmdk/README.rst create mode 100755 examples/gpmdk/build_chicoma_hackathon.sh create mode 100644 examples/gpmdk/docs/Makefile create mode 100644 examples/gpmdk/docs/source-doxygen/gpmd_doxyfile create mode 100644 examples/gpmdk/docs/source-sphinx/Makefile create mode 100644 examples/gpmdk/docs/source-sphinx/TUTORIAL.rst create mode 100644 examples/gpmdk/docs/source-sphinx/conf.py create mode 100644 examples/gpmdk/docs/source-sphinx/index.rst create mode 100644 examples/gpmdk/run/coords_2088.dat create mode 100644 examples/gpmdk/run/coords_300.dat create mode 100755 examples/gpmdk/run/get_energy.py create mode 100644 examples/gpmdk/run/gpmd_input.in create mode 100644 examples/gpmdk/run/input.in create mode 100755 examples/gpmdk/run/job create mode 100644 examples/gpmdk/run/latte.in create mode 100755 examples/gpmdk/run/run.sh create mode 100755 examples/gpmdk/run/sbatch.sh create mode 100644 examples/gpmdk/run/wrapper.sh create mode 100644 examples/gpmdk/setenv_chicoma_hackathon.sh create mode 100644 examples/gpmdk/src/CMakeLists.txt create mode 100755 examples/gpmdk/src/gpmd.py create mode 100644 examples/gpmdk/src/gpmd_secuential.F90 create mode 100644 examples/gpmdk/src/gpmdcov_allocation.F90 create mode 100644 examples/gpmdk/src/gpmdcov_assert.F90 create mode 100644 examples/gpmdk/src/gpmdcov_buildz.F90 create mode 100644 examples/gpmdk/src/gpmdcov_constraints.F90 create mode 100644 examples/gpmdk/src/gpmdcov_diagonalize.F90 create mode 100644 examples/gpmdk/src/gpmdcov_dm_min.F90 create mode 100644 examples/gpmdk/src/gpmdcov_dump.F90 create mode 100644 examples/gpmdk/src/gpmdcov_energandforces.F90 create mode 100644 examples/gpmdk/src/gpmdcov_field.F90 create mode 100644 examples/gpmdk/src/gpmdcov_finalize.F90 create mode 100644 examples/gpmdk/src/gpmdcov_firstcharges.F90 create mode 100644 examples/gpmdk/src/gpmdcov_graphpart.F90 create mode 100644 examples/gpmdk/src/gpmdcov_highorder.F90 create mode 100644 examples/gpmdk/src/gpmdcov_init.F90 create mode 100644 examples/gpmdk/src/gpmdcov_initparts.F90 create mode 100644 examples/gpmdk/src/gpmdcov_kernel.F90 create mode 100644 examples/gpmdk/src/gpmdcov_langevin.F90 create mode 100644 examples/gpmdk/src/gpmdcov_latte.F90 create mode 100644 examples/gpmdk/src/gpmdcov_lib.F90 create mode 100644 examples/gpmdk/src/gpmdcov_lib_mod.F90 create mode 100644 examples/gpmdk/src/gpmdcov_mdloop.F90 create mode 100644 examples/gpmdk/src/gpmdcov_mod.F90 create mode 100644 examples/gpmdk/src/gpmdcov_mod_orig.F90 create mode 100644 examples/gpmdk/src/gpmdcov_neighbor.F90 create mode 100644 examples/gpmdk/src/gpmdcov_nonequilibrium.F90 create mode 100644 examples/gpmdk/src/gpmdcov_parser.F90 create mode 100644 examples/gpmdk/src/gpmdcov_part.F90 create mode 100644 examples/gpmdk/src/gpmdcov_preparemd.F90 create mode 100644 examples/gpmdk/src/gpmdcov_rankN.F90 create mode 100644 examples/gpmdk/src/gpmdcov_reshuffle.F90 create mode 100644 examples/gpmdk/src/gpmdcov_restart.F90 create mode 100644 examples/gpmdk/src/gpmdcov_rhosolver.F90 create mode 100644 examples/gpmdk/src/gpmdcov_test.F90 create mode 100644 examples/gpmdk/src/gpmdcov_vars.F90 create mode 100644 examples/gpmdk/src/gpmdcov_writeout.F90 create mode 100644 examples/gpmdk/src/gpmdk.F90 create mode 100755 examples/gpmdk/tests/get_energy.py create mode 100644 examples/gpmdk/tests/input_BlockPart.in create mode 100644 examples/gpmdk/tests/input_DiagEf.in create mode 100644 examples/gpmdk/tests/input_KerByPartsBuildAlwaysMetis.in create mode 100644 examples/gpmdk/tests/input_KerByPartsBuildOnceBlock.in create mode 100644 examples/gpmdk/tests/input_KerByPartsBuildOnceBlockUpdate.in create mode 100644 examples/gpmdk/tests/input_KerByPartsBuildOnceMetisUpdate.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildAlways.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildAlwaysBlock.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildAlwaysMetis.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildOnce.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildOnceBlock.in create mode 100644 examples/gpmdk/tests/input_KerFullBuildOnceMetis.in create mode 100644 examples/gpmdk/tests/input_KerUpdate.in create mode 100644 examples/gpmdk/tests/input_test.in create mode 100755 examples/gpmdk/tests/latteTBparams/bondints.nonortho create mode 100755 examples/gpmdk/tests/latteTBparams/bondints.nonortho.algo create mode 100755 examples/gpmdk/tests/latteTBparams/bondints.ortho create mode 100755 examples/gpmdk/tests/latteTBparams/electrons.dat.algo create mode 100755 examples/gpmdk/tests/latteTBparams/ppots.nonortho create mode 100755 examples/gpmdk/tests/latteTBparams/ppots.nonortho.algo create mode 100755 examples/gpmdk/tests/latteTBparams/ppots.ortho create mode 100644 examples/gpmdk/tests/ref.BlockPart create mode 100644 examples/gpmdk/tests/ref.DiagEf create mode 100644 examples/gpmdk/tests/ref.KerUpdate create mode 100644 examples/gpmdk/tests/ref.gpmdcov_DM_Min create mode 100644 examples/gpmdk/tests/ref.gpmdcov_FirstCharges create mode 100644 examples/gpmdk/tests/ref.gpmdcov_Init create mode 100644 examples/gpmdk/tests/ref.gpmdcov_InitParts create mode 100644 examples/gpmdk/tests/ref.gpmdcov_MDloop create mode 100644 examples/gpmdk/tests/ref.gpmdcov_Part create mode 100644 examples/gpmdk/tests/ref.gpmdcov_PrepareMD create mode 100755 examples/gpmdk/tests/run_test.sh create mode 100755 examples/gpmdk/tests/test-energy.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..012f1687 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "bml"] + path = bml + url = https://github.com/lanl/bml + branch = hackathon diff --git a/bml b/bml new file mode 160000 index 00000000..0e94c8e2 --- /dev/null +++ b/bml @@ -0,0 +1 @@ +Subproject commit 0e94c8e25892bbd48b94603a9a204d3d73eec85e diff --git a/examples/gpmdk/README.rst b/examples/gpmdk/README.rst new file mode 100644 index 00000000..9cb5ddaa --- /dev/null +++ b/examples/gpmdk/README.rst @@ -0,0 +1,97 @@ +Graph-partition Quantum based Molecular Dynamics with Kernel (GPMDK) +======================================================= + +About +******** +The following research code is based on techniques that were recently published in `J. Chem. Phys. 158, 074108 (2023) `_. +Briefly, the graph-based electronic structure theory is applied by using the atomic graph and Quantum Molecular Dynamics is performed on a completely memory distributed way. Moreover, this code make use of two LANL developed libraries, namely `PROGRESS `_ and `BML `_ which need to be properly installed in order to build the GPMD code. More about this libraries can be found in the following references `arXiv:2401.13772 `_ and `J.Supercomput. 74 (11): 6201–19. `_. + + +Getting started +****************** +We assume that you have access to the LANL Institutional Computing (IC) as well as access to Darwin. In order to get a Darwin account, please visit the following +`lik `_. +For Institutional computing and getting access to Chicoma, please, ask the project manager (whoever got +time to run on IC machines by submitting a proposal) to include you in the specific +project. Then verify that you are properly included in `here `_. + +This material can be cloned or downloaded from the PROGRESS github repository (`github link `_). +The repository can be cloned as follows: + + .. code-block:: bash + + git clone https://github.com/lanl/qmd-progress.git + +Look in the examples/gpmdk subdirectory. + +Further build instructions for gpmdk to be written + +**Building required packages METIS and Magma** + +Download METIS (latest tested version: 5.1.0): +http://glaros.dtc.umn.edu/gkhome/metis/metis/download +SCP and unpack using: + .. code-block:: bash + scp metis-5.1.0.tar.gz username@ch-fe1:. + tar -xvf metis-5.1.0.tar.gz + +Build METIS: +// Move into METIS directory + .. code-block:: bash + cd metis-5.1.0/ +// -cc is the C compiler to use +//-prefix is where to build the metis libraries and can be changed + .. code-block:: bash + make config -shared=1 -cc=cc -prefix=~/local + make install + +Download Magma (currently using version 2.5.4) from MAGMA website: +https://icl.utk.edu/magma/downloads/ + +Secure copy magma tar ball to chicoma: +// Replace “username” with your user name + .. code-block:: bash + scp magma-2.5.4.tar.gz username@ch-fe1:. + +Untar and unzip the tarball and rename the directory: + .. code-block:: bash + tar -xvf magma-2.5.4.tar.gz + mv magma-2.5.4/ magma + +Build magma using the “build_magma.sh” script provided in gpmd/chicomaGPU: +// Start outside of the magma directory then type the following command: + .. code-block:: bash + bash build_magma.sh + +**Building BML and QMD-PROGRESS** + +Clone into the BML repository: + .. code-block:: bash + git clone https://github.com/lanl/bml.git +Build BML by running the build_bml.sh script provided in gpmd/chicomaGPU in the directory immediately above bml/ : + .. code-block:: bash + bash build_bml.sh +BML requires a BLAS library (basic linear algebra subprograms) which should be available via the intel-mkl module. If you run into issues with BLAS not being found, check to make sure the intel-mkl module is loaded correctly (this should have been done by the setup-envs.sh script). You can check which modules are loaded using the following command: + .. code-block:: bash + module list +Clone into the QMD-PROGRESS repository: + .. code-block:: bash + git clone https://github.com/lanl/qmd-progress.git +Build QMD-PROGRESS by running the build_progress.sh script provided in gpmd/chicomaGPU in the directory immediately above qmd-progress: +// ** Note: QMD-PROGRESS requires BML, so be sure to build BML first + .. code-block:: bash + bash build_progress.sh + +**Building GPMDK** + +Build GPMDK after building all other dependencies (METIS, Magma, BML, and QMD-PROGRESS) by running the build_cmake.sh script provided with GPMDK : + .. code-block:: bash + bash build_cmake.sh + +Running GPMDK +******************* + +Go into the ``run`` folder. There is an input file ``input.in`` providing most of the variables that are needed by the code. There +are also several ``pdb`` files with chemical systems that one can use as examples. + + diff --git a/examples/gpmdk/build_chicoma_hackathon.sh b/examples/gpmdk/build_chicoma_hackathon.sh new file mode 100755 index 00000000..b04c9c4b --- /dev/null +++ b/examples/gpmdk/build_chicoma_hackathon.sh @@ -0,0 +1,10 @@ +WORKDIR=/usr/projects/icapt/mewall/packages/gpmd +rm -rf build_hackathon +mkdir build_hackathon +cd build_hackathon +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" \ +-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero" \ +-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/" ../src/ +#-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ +make + diff --git a/examples/gpmdk/docs/Makefile b/examples/gpmdk/docs/Makefile new file mode 100644 index 00000000..2d6a4f46 --- /dev/null +++ b/examples/gpmdk/docs/Makefile @@ -0,0 +1,19 @@ +# Makefile for the GPMD documentation + +DIR=$(shell pwd) + +all: + (make doxy) + +clean: + rm -f *.backup + +doxy: + (doxygen ./source-doxygen/gpmd_doxyfile) + (cd ./build-doxygen/latex ; make ; cp refman.pdf ../../gpmd-doxygen.pdf) + +sphinx: + (cp ../README.rst ./source-sphinx/README.rst) + (cd ./source-sphinx/; make latex) + (cd ./build-sphinx/latex/; make; cp gpmd.pdf ../../gpmd-sphinx.pdf) + diff --git a/examples/gpmdk/docs/source-doxygen/gpmd_doxyfile b/examples/gpmdk/docs/source-doxygen/gpmd_doxyfile new file mode 100644 index 00000000..d9647b08 --- /dev/null +++ b/examples/gpmdk/docs/source-doxygen/gpmd_doxyfile @@ -0,0 +1,407 @@ +# Doxyfile 1.9.8 + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = "GPMD" +PROJECT_NUMBER = +PROJECT_BRIEF = +PROJECT_LOGO = +OUTPUT_DIRECTORY = ./build-doxygen +CREATE_SUBDIRS = NO +CREATE_SUBDIRS_LEVEL = 8 +ALLOW_UNICODE_NAMES = NO +OUTPUT_LANGUAGE = English +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = NO +JAVADOC_BANNER = NO +QT_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +PYTHON_DOCSTRING = YES +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 4 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +OPTIMIZE_OUTPUT_SLICE = NO +EXTENSION_MAPPING = +MARKDOWN_SUPPORT = YES +TOC_INCLUDE_HEADINGS = 5 +MARKDOWN_ID_STYLE = DOXYGEN +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = NO +GROUP_NESTED_COMPOUNDS = NO +SUBGROUPING = YES +INLINE_GROUPED_CLASSES = NO +INLINE_SIMPLE_STRUCTS = NO +TYPEDEF_HIDES_STRUCT = NO +LOOKUP_CACHE_SIZE = 0 +NUM_PROC_THREADS = 1 +TIMESTAMP = NO +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = NO +EXTRACT_PRIVATE = NO +EXTRACT_PRIV_VIRTUAL = NO +EXTRACT_PACKAGE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +RESOLVE_UNNAMED_PARAMS = YES +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = SYSTEM +HIDE_SCOPE_NAMES = NO +HIDE_COMPOUND_REFERENCE= NO +SHOW_HEADERFILE = YES +SHOW_INCLUDE_FILES = YES +SHOW_GROUPED_MEMB_INC = NO +FORCE_LOCAL_INCLUDES = NO +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = NO +SORT_MEMBERS_CTORS_1ST = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +STRICT_PROTO_MATCHING = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = YES +FILE_VERSION_FILTER = +LAYOUT_FILE = +CITE_BIB_FILES = +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_IF_INCOMPLETE_DOC = YES +WARN_NO_PARAMDOC = NO +WARN_IF_UNDOC_ENUM_VAL = NO +WARN_AS_ERROR = NO +WARN_FORMAT = "$file:$line: $text" +WARN_LINE_FORMAT = "at line $line of file $file" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = ../../src/ +INPUT_ENCODING = UTF-8 +INPUT_FILE_ENCODING = +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cxxm \ + *.cpp \ + *.cppm \ + *.c++ \ + *.c++m \ + *.F90 \ + *.f90 \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.ixx \ + *.l \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice +RECURSIVE = NO +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = * +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +FILTER_SOURCE_PATTERNS = +USE_MDFILE_AS_MAINPAGE = +FORTRAN_COMMENT_AFTER = 72 +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = NO +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = NO +REFERENCES_RELATION = NO +REFERENCES_LINK_SOURCE = YES +SOURCE_TOOLTIPS = YES +USE_HTAGS = NO +VERBATIM_HEADERS = YES +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = YES +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_EXTRA_STYLESHEET = +HTML_EXTRA_FILES = +HTML_COLORSTYLE = AUTO_LIGHT +HTML_COLORSTYLE_HUE = 220 +HTML_COLORSTYLE_SAT = 100 +HTML_COLORSTYLE_GAMMA = 80 +HTML_DYNAMIC_MENUS = YES +HTML_DYNAMIC_SECTIONS = NO +HTML_CODE_FOLDING = YES +HTML_INDEX_NUM_ENTRIES = 100 +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_FEEDURL = +DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_NAME = Publisher +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = NO +SITEMAP_URL = +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = org.doxygen.Project +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +GENERATE_ECLIPSEHELP = NO +ECLIPSE_DOC_ID = org.doxygen.Project +DISABLE_INDEX = NO +GENERATE_TREEVIEW = NO +FULL_SIDEBAR = NO +ENUM_VALUES_PER_LINE = 4 +TREEVIEW_WIDTH = 250 +EXT_LINKS_IN_WINDOW = NO +OBFUSCATE_EMAILS = YES +HTML_FORMULA_FORMAT = png +FORMULA_FONTSIZE = 10 +FORMULA_MACROFILE = +USE_MATHJAX = NO +MATHJAX_VERSION = MathJax_2 +MATHJAX_FORMAT = HTML-CSS +MATHJAX_RELPATH = +MATHJAX_EXTENSIONS = +MATHJAX_CODEFILE = +SEARCHENGINE = YES +SERVER_BASED_SEARCH = NO +EXTERNAL_SEARCH = NO +SEARCHENGINE_URL = +SEARCHDATA_FILE = searchdata.xml +EXTERNAL_SEARCH_ID = +EXTRA_SEARCH_MAPPINGS = +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = YES +LATEX_OUTPUT = latex +LATEX_CMD_NAME = +MAKEINDEX_CMD_NAME = makeindex +LATEX_MAKEINDEX_CMD = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4 +EXTRA_PACKAGES = +LATEX_HEADER = +LATEX_FOOTER = +LATEX_EXTRA_STYLESHEET = +LATEX_EXTRA_FILES = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +LATEX_BIB_STYLE = plain +LATEX_EMOJI_DIRECTORY = +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_SUBDIR = +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_PROGRAMLISTING = YES +XML_NS_MEMB_FILE_SCOPE = NO +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- +GENERATE_DOCBOOK = NO +DOCBOOK_OUTPUT = docbook +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- +GENERATE_SQLITE3 = NO +SQLITE3_OUTPUT = sqlite3 +SQLITE3_RECREATE_DB = YES +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +EXTERNAL_PAGES = YES +#--------------------------------------------------------------------------- +# Configuration options related to diagram generator tools +#--------------------------------------------------------------------------- +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = NO +DOT_NUM_THREADS = 0 +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +UML_LIMIT_NUM_FIELDS = 10 +DOT_UML_DETAILS = NO +DOT_WRAP_THRESHOLD = 17 +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +CALLER_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DIR_GRAPH_MAX_DEPTH = 1 +DOT_IMAGE_FORMAT = png +INTERACTIVE_SVG = NO +DOT_PATH = +DOTFILE_DIRS = +DIA_PATH = +DIAFILE_DIRS = +PLANTUML_JAR_PATH = +PLANTUML_CFG_FILE = +PLANTUML_INCLUDE_PATH = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES +MSCGEN_TOOL = +MSCFILE_DIRS = diff --git a/examples/gpmdk/docs/source-sphinx/Makefile b/examples/gpmdk/docs/source-sphinx/Makefile new file mode 100644 index 00000000..11fc5f6f --- /dev/null +++ b/examples/gpmdk/docs/source-sphinx/Makefile @@ -0,0 +1,21 @@ +#minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build-3.12 +SOURCEDIR = ../source-sphinx +BUILDDIR = ../build-sphinx + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + diff --git a/examples/gpmdk/docs/source-sphinx/TUTORIAL.rst b/examples/gpmdk/docs/source-sphinx/TUTORIAL.rst new file mode 100644 index 00000000..0181cbbe --- /dev/null +++ b/examples/gpmdk/docs/source-sphinx/TUTORIAL.rst @@ -0,0 +1,13 @@ +Tutorial +========== + + +Steered MD +############## + +Ad tut here + + + .. code-block:: bash + + STMR0= diff --git a/examples/gpmdk/docs/source-sphinx/conf.py b/examples/gpmdk/docs/source-sphinx/conf.py new file mode 100644 index 00000000..071934b0 --- /dev/null +++ b/examples/gpmdk/docs/source-sphinx/conf.py @@ -0,0 +1,28 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'gpmd' +copyright = '2024, GPMD team' +author = 'GPMD team' +release = '1.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ['_templates'] +exclude_patterns = [] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/examples/gpmdk/docs/source-sphinx/index.rst b/examples/gpmdk/docs/source-sphinx/index.rst new file mode 100644 index 00000000..b39a58b8 --- /dev/null +++ b/examples/gpmdk/docs/source-sphinx/index.rst @@ -0,0 +1,22 @@ +.. gpmd documentation master file, created by + sphinx-quickstart on Wed Mar 13 15:41:17 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to gpmd's documentation! +================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + ./README.rst + ./TUTORIAL.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/examples/gpmdk/run/coords_2088.dat b/examples/gpmdk/run/coords_2088.dat new file mode 100644 index 00000000..e0a6c820 --- /dev/null +++ b/examples/gpmdk/run/coords_2088.dat @@ -0,0 +1,2092 @@ + 2088 + 27.83000 0.00000 0.00000 + 0.00000 27.83000 0.00000 + 0.00000 0.00000 27.83000 + O 13.72700 13.73400 13.91500 + H 13.30900 14.59500 13.91500 + H 14.70800 13.41500 13.91500 +O 2.30000 6.28000 1.13000 +H 1.37000 6.26000 1.50000 +H 2.31000 5.89000 0.21000 +O 5.69000 12.75100 11.65100 +H 4.76000 12.68100 11.28100 +H 5.80000 13.64100 12.09100 +O 15.55100 15.11100 7.03000 +H 14.98100 14.95100 7.84000 +H 14.96100 15.21100 6.23000 +O 17.43100 6.18000 8.56000 +H 17.76100 7.12000 8.56000 +H 17.94100 5.64000 9.22000 +O 11.35100 7.03000 7.17000 +H 11.92100 7.81000 6.92000 +H 10.75100 7.29000 7.93000 +O 17.55100 6.07000 2.31000 +H 17.43100 5.94000 1.32000 +H 17.25100 5.26000 2.80000 +O 7.68000 11.44100 10.23100 +H 6.90000 11.61100 10.83100 +H 8.02000 12.31100 9.87100 +O 8.50000 7.98000 18.23100 +H 8.46000 8.74000 18.88100 +H 8.72000 8.34000 17.32100 +O 6.85000 10.12100 6.65000 +H 7.54000 9.96100 7.35000 +H 6.12000 10.69100 7.03000 +O 6.86000 11.61100 18.03100 +H 7.46000 12.40100 18.17100 +H 6.00000 11.92100 17.62100 +O 3.35000 14.35100 10.61100 +H 2.57000 14.04100 10.08100 +H 3.93000 14.93100 10.04100 +O 4.38000 3.92000 14.99100 +H 5.20000 3.36000 15.08100 +H 3.57000 3.34000 15.03100 +O 16.03100 4.47000 7.37000 +H 15.29100 4.93000 6.87000 +H 16.54100 5.15000 7.90000 +O 2.31000 17.13100 4.83000 +H 2.65000 17.90100 5.37000 +H 2.75000 17.13100 3.93000 +O 11.27100 13.41100 16.90100 +H 11.74100 13.41100 17.78100 +H 10.79100 12.54100 16.79100 +O 2.40000 10.91100 8.86000 +H 2.54000 10.07100 9.38000 +H 1.85000 11.55100 9.41000 +O 6.20000 17.86100 14.39100 +H 5.28000 17.69100 14.74100 +H 6.48000 18.78100 14.65100 +O 6.06000 9.64100 1.23000 +H 6.13000 10.48100 0.69000 +H 6.52000 9.77100 2.11000 +O 15.94100 1.14000 14.80100 +H 15.76100 1.81000 14.08100 +H 15.91100 1.60000 15.69100 +O 18.42100 17.67100 3.59000 +H 18.96100 17.38100 4.39000 +H 18.72100 18.57100 3.30000 +O 16.89100 9.22000 6.12000 +H 17.84100 8.93000 6.20000 +H 16.81100 9.87000 5.37000 +O 16.41100 11.08100 4.32000 +H 17.27100 11.10100 3.80000 +H 16.55100 11.55100 5.20000 +O 6.13000 13.65100 7.26000 +H 5.64000 12.78100 7.35000 +H 5.90000 14.08100 6.39000 +O 8.09000 0.04000 5.02000 +H 8.49000 0.95000 4.93000 +H 7.09000 0.12000 5.08000 +O 15.25100 9.99100 1.90000 +H 14.62100 9.23100 2.03000 +H 15.73100 10.17100 2.76000 +O 11.87100 17.92100 16.16100 +H 12.11100 18.52100 15.40100 +H 11.94100 16.97100 15.86100 +O 3.17000 2.51000 18.01100 +H 3.88000 3.22000 18.07100 +H 2.29000 2.90000 18.29100 +O 14.66100 14.17100 9.53100 +H 14.07100 14.23100 10.33100 +H 14.51100 13.29100 9.07100 +O 16.67100 17.14100 5.72000 +H 16.26100 16.91100 4.84000 +H 16.49100 16.40100 6.37000 +O 5.98000 7.29000 2.70000 +H 6.22000 7.98000 2.02000 +H 5.20000 7.62000 3.24000 +O 12.81100 3.45000 9.44100 +H 11.95100 2.95000 9.31100 +H 13.43100 2.91000 10.00100 +O 15.76100 16.62100 3.07000 +H 16.65100 17.08100 3.10000 +H 15.55100 16.38100 2.12000 +O 8.07000 6.05000 14.65100 +H 7.60000 6.02000 15.54100 +H 7.56000 5.50000 13.99100 +O 13.94100 4.69000 16.74100 +H 13.74100 5.12000 17.62100 +H 14.72100 4.07000 16.83100 +O 9.73100 8.90000 15.72100 +H 10.19100 8.06000 15.43100 +H 9.17100 9.24000 14.97100 +O 9.91100 4.10000 12.42100 +H 9.14100 4.44000 12.96100 +H 9.57100 3.59000 11.63100 +O 10.41100 7.01000 4.29000 +H 10.67100 6.97000 5.25000 +H 9.56100 6.50000 4.15000 +O 8.65000 3.48000 1.95000 +H 9.24000 4.11000 1.46000 +H 8.84000 2.54000 1.66000 +O 17.19100 5.85000 18.31100 +H 16.93100 6.74000 17.95100 +H 17.17100 5.17000 17.58100 +O 13.62100 11.44100 5.45000 +H 14.45100 11.15100 4.97000 +H 13.13100 12.11100 4.89000 +O 5.50000 1.96000 8.85000 +H 5.45000 1.91000 9.85000 +H 5.52000 2.92000 8.56000 +O 10.08100 14.56100 4.77000 +H 9.62100 15.28100 4.25000 +H 10.04100 14.76100 5.75000 +O 3.51000 18.01100 8.53000 +H 4.01000 17.15100 8.59000 +H 4.16000 18.78100 8.50000 +O 17.95100 10.66100 8.73000 +H 17.33100 10.51100 7.97000 +H 17.43100 10.77100 9.58000 +O 12.27100 15.50100 15.06100 +H 12.33100 14.73100 15.70100 +H 11.75100 15.24100 14.26100 +O 3.21000 9.43100 2.42000 +H 4.03000 9.82100 2.00000 +H 2.94000 8.61100 1.93000 +O 14.58100 7.35000 7.28000 +H 14.53100 6.70000 8.03000 +H 15.38100 7.94000 7.41000 +O 4.61000 12.66100 17.27100 +H 4.11000 12.67100 16.41100 +H 3.98000 12.48100 18.03100 +O 11.11100 17.76100 2.37000 +H 10.51100 17.14100 2.87000 +H 11.42100 17.32100 1.52000 +O 2.02000 2.85000 14.98100 +H 1.22000 3.45000 14.85100 +H 1.92000 2.36000 15.84100 +O 16.32100 13.77100 0.81000 +H 16.00100 14.71100 0.71000 +H 15.56100 13.14100 0.69000 +O 4.64000 17.43100 3.23000 +H 4.97000 17.82100 4.09000 +H 5.40000 17.36100 2.58000 +O 14.00100 1.07000 4.26000 +H 13.76100 0.70000 3.36000 +H 14.99100 1.23000 4.30000 +O 9.40100 16.98100 9.04000 +H 10.20100 16.41100 9.25000 +H 8.91100 16.58100 8.27000 +O 3.82000 7.00000 4.80000 +H 4.27000 6.10000 4.77000 +H 2.88000 6.89000 5.13000 +O 15.47100 2.22000 17.29100 +H 15.42100 2.59000 18.21100 +H 14.75100 1.53000 17.17100 +O 6.14000 1.22000 1.17000 +H 7.12000 1.00000 1.24000 +H 5.83000 1.05000 0.24000 +O 7.81000 2.64000 17.49100 +H 8.48000 2.03000 17.92100 +H 7.08000 2.83000 18.14100 +O 8.88000 15.14100 11.95100 +H 8.65000 14.89100 11.01100 +H 9.49000 14.45100 12.34100 +O 13.51100 5.90000 14.33100 +H 13.79100 5.47000 15.18100 +H 13.76100 6.86000 14.34100 +O 8.03000 14.02100 9.24000 +H 8.93000 14.16100 8.82000 +H 7.32000 14.04100 8.53000 +O 9.22000 5.03000 8.99000 +H 8.97000 4.94000 8.03000 +H 9.70000 4.21000 9.30000 +O 5.39000 0.64000 5.12000 +H 4.58000 0.65000 5.70000 +H 5.42000 1.47000 4.57000 +O 14.34100 11.88100 0.41000 +H 14.66100 11.12100 0.98000 +H 13.42100 12.15100 0.71000 +O 2.97000 0.35000 1.71000 +H 3.46000 1.19000 1.50000 +H 3.59000 -0.30000 2.16000 +O 9.35100 2.36000 4.80000 +H 8.87100 2.77000 4.02000 +H 10.34100 2.34000 4.61000 +O 10.76100 6.83000 14.64100 +H 9.96100 6.22000 14.67100 +H 11.57100 6.30000 14.40100 +O 12.27100 15.70100 7.93000 +H 12.48100 16.44100 7.28000 +H 12.95100 15.70100 8.66000 +O 4.59000 11.52100 7.41000 +H 3.88000 11.25100 8.06000 +H 4.33000 11.24100 6.48000 +O 12.71100 17.97100 5.91000 +H 13.15100 18.61100 5.27000 +H 12.21100 18.49100 6.61000 +O 10.32100 5.49000 0.16000 +H 9.91100 6.31000 -0.23000 +H 10.96100 5.75000 0.89000 +O 0.78000 5.56000 13.86100 +H 1.70000 5.55000 13.45100 +H 0.72000 6.30000 14.53100 +O 5.61000 2.22000 11.47100 +H 5.99000 1.38000 11.84100 +H 4.73000 2.41000 11.91100 +O 8.66000 4.54000 6.42000 +H 8.34000 5.26000 5.80000 +H 8.90000 3.73000 5.89000 +O 10.17100 0.39000 7.53000 +H 9.45100 0.44000 6.84000 +H 9.93100 -0.30000 8.22000 +O 14.29100 11.73100 8.67000 +H 13.74100 10.89100 8.60000 +H 14.55100 12.02100 7.75000 +O 14.66100 5.90000 9.92100 +H 14.36100 4.95000 9.99100 +H 15.39100 6.06000 10.58100 +O 18.57100 8.33000 3.77000 +H 18.99100 7.69000 4.41000 +H 18.19100 7.82000 2.99000 +O 4.88000 13.85100 1.74000 +H 4.01000 13.70100 2.21000 +H 4.71000 14.11100 0.79000 +O 16.64100 12.80100 6.57000 +H 17.63100 12.88100 6.71000 +H 16.19100 13.64100 6.88000 +O 13.90100 5.75000 0.78000 +H 13.36100 5.54000 1.59000 +H 14.81100 5.34000 0.87000 +O 5.27000 2.56000 3.28000 +H 5.54000 1.97000 2.53000 +H 5.27000 3.51000 2.97000 +O 17.54100 12.23100 15.88100 +H 18.45100 11.84100 15.75100 +H 17.62100 13.19100 16.12100 +O 10.64100 13.47100 13.40100 +H 9.84100 13.24100 13.95100 +H 11.47100 13.21100 13.89100 +O 15.92100 16.29100 16.25100 +H 16.19100 16.63100 15.35100 +H 16.71100 15.91100 16.71100 +O 11.11100 11.95100 11.00100 +H 10.71100 12.39100 11.81100 +H 10.70100 12.32100 10.17100 +O 16.38100 10.99100 10.79100 +H 16.43100 11.80100 11.38100 +H 15.52100 11.01100 10.28100 +O 9.15000 0.89000 14.02100 +H 9.40000 0.69000 13.07100 +H 9.87000 1.45000 14.44100 +O 9.80100 11.16100 17.19100 +H 8.81100 11.22100 17.29100 +H 10.03100 10.36100 16.63100 +O 7.05000 10.50100 3.68000 +H 6.91000 10.57100 4.67000 +H 7.89000 9.99100 3.50000 +O 4.10000 8.13000 12.51100 +H 4.96000 8.25000 13.01100 +H 3.68000 7.26000 12.78100 +O 12.74100 3.86000 12.62100 +H 12.95100 4.60000 13.26100 +H 11.85100 4.03000 12.19100 +O 3.67000 11.00100 5.01000 +H 3.60000 11.83100 4.45000 +H 3.71000 10.20100 4.41000 +O 5.66000 5.37000 8.65000 +H 5.78000 6.03000 7.91000 +H 6.12000 5.71000 9.48000 +O 12.52100 13.48100 3.88000 +H 13.02100 14.25100 4.28000 +H 11.57100 13.50100 4.20000 +O 12.72100 14.45100 11.42100 +H 13.19100 14.58100 12.29100 +H 12.06100 13.71100 11.51100 +O 15.82100 6.39000 4.72000 +H 15.51100 7.00000 5.45000 +H 16.32100 6.91000 4.03000 +O 3.54000 15.10100 13.29100 +H 3.33000 14.66100 12.42100 +H 4.51000 15.36100 13.32100 +O 4.02000 7.51000 15.98100 +H 4.70000 8.06000 15.51100 +H 4.42000 6.63000 16.25100 +O 15.87100 7.79000 16.70100 +H 14.95100 8.17000 16.65100 +H 16.47100 8.26000 16.05100 +O 10.13100 1.05000 17.70100 +H 10.19100 1.90000 17.18100 +H 10.45100 0.29000 17.13100 +O 5.04000 0.50000 17.40100 +H 4.62000 -0.07000 16.70100 +H 4.38000 1.19000 17.72100 +O 5.73000 8.70000 10.29100 +H 6.17000 9.59000 10.20100 +H 5.10000 8.70000 11.06100 +O 13.60100 8.62000 10.45100 +H 12.85100 8.62000 9.79100 +H 13.97100 7.70000 10.54100 +O 12.09100 5.25000 2.75000 +H 12.22100 4.41000 3.29000 +H 11.80100 5.99000 3.35000 +O 3.07000 2.13000 12.31100 +H 2.84000 2.50000 13.21100 +H 2.77000 1.18000 12.25100 +O 0.37000 13.10100 12.82100 +H 0.90000 12.61100 13.50100 +H 0.59000 14.08100 12.87100 +O 7.32000 6.34000 10.64100 +H 7.91000 6.08000 9.88100 +H 7.04000 7.30000 10.53100 +O 17.28100 9.35100 18.54100 +H 16.82100 9.28100 17.65100 +H 16.66100 9.79100 19.20100 +O 16.22100 3.67000 3.74000 +H 16.24100 2.91000 4.38000 +H 15.74100 4.44000 4.14000 +O 10.23100 7.66000 9.66100 +H 10.38100 7.87000 10.62100 +H 9.93100 6.71000 9.57100 +O 9.80100 15.73100 17.00100 +H 9.60100 16.17100 16.12100 +H 10.19100 14.82100 16.84100 +O 18.59100 15.18100 16.05100 +H 18.73100 15.45100 15.10100 +H 19.42100 15.40100 16.58100 +O 3.50000 8.98000 18.04100 +H 4.26000 9.42000 18.52100 +H 3.85000 8.51000 17.22100 +O 15.40100 2.74000 1.25000 +H 14.79100 1.99000 1.48000 +H 15.62100 3.26000 2.08000 +O 13.03100 8.38000 0.42000 +H 13.37100 7.45000 0.57000 +H 13.21100 8.65000 -0.53000 +O 10.68100 13.33100 8.49000 +H 10.75100 12.49100 7.94000 +H 11.30100 14.02100 8.13000 +O 3.19000 8.10000 9.49100 +H 4.12000 8.46000 9.54100 +H 3.13000 7.25000 10.01100 +O 3.39000 5.09000 10.06100 +H 2.87000 4.26000 9.89100 +H 4.16000 5.14000 9.42100 +O 11.38100 3.80000 16.78100 +H 10.93100 4.43000 17.42100 +H 12.31100 4.11000 16.61100 +O 11.60100 2.07000 14.77100 +H 11.60100 2.71000 15.54100 +H 11.88100 2.55000 13.94100 +O 5.04000 15.68100 9.10000 +H 5.70000 16.42100 9.19000 +H 5.48000 14.89100 8.68000 +O 10.02100 7.96000 12.38100 +H 10.43100 7.64000 13.24100 +H 9.06100 7.69000 12.35100 +O 0.40000 5.44000 11.14100 +H 1.25000 5.11000 10.73100 +H 0.53000 5.59000 12.12100 +O 1.89000 5.20000 17.22100 +H 2.48000 4.80000 16.52100 +H 1.31000 5.91000 16.81100 +O 13.69100 9.50100 16.60100 +H 14.08100 10.39100 16.80100 +H 13.79100 9.30100 15.63100 +O 8.15000 5.72000 3.25000 +H 8.22000 4.83000 2.79000 +H 7.21000 6.06000 3.17000 +O 16.57100 6.04000 12.06100 +H 16.19100 5.35000 12.68100 +H 17.39100 5.68000 11.62100 +O 2.52000 15.64100 17.44100 +H 2.22000 16.21100 18.20100 +H 2.45000 14.67100 17.70100 +O 6.71000 4.64000 12.69100 +H 6.37000 3.75000 12.39100 +H 6.97000 5.18000 11.89100 +O 9.30000 16.78100 14.65100 +H 9.06000 16.60100 13.70100 +H 9.60000 17.72100 14.75100 +O 4.73000 5.00000 1.91000 +H 5.34000 5.80000 1.95000 +H 3.78000 5.31000 1.98000 +O 1.59000 11.37100 14.66100 +H 1.81000 10.76100 15.42100 +H 1.69000 10.88100 13.80100 +O 13.47100 10.59100 12.34100 +H 13.71100 9.96100 11.60100 +H 12.57100 10.99100 12.16100 +O 13.02100 8.55000 3.09000 +H 12.16100 8.24000 3.51000 +H 12.98100 8.41000 2.10000 +O 17.59100 17.47100 11.54100 +H 18.20100 17.77100 10.81100 +H 17.21100 16.58100 11.32100 +O 12.52100 17.31100 11.28100 +H 13.36100 17.36100 10.74100 +H 12.29100 16.35100 11.46100 +O 0.83000 12.58100 10.22100 +H 0.78000 12.57100 11.22100 +H 0.00000 12.17100 9.84100 +O 6.88000 16.62100 17.16100 +H 6.32000 17.43100 17.25100 +H 7.40000 16.66100 16.30100 +O 9.03000 0.86000 1.33000 +H 9.54000 0.87000 0.47000 +H 9.59000 0.44000 2.04000 +O 17.26100 1.35000 5.23000 +H 17.99100 1.18000 4.56000 +H 16.95100 0.48000 5.61000 +O 13.88100 15.73100 4.77000 +H 14.55100 15.85100 4.03000 +H 13.48100 16.62100 5.00000 +O 1.30000 17.94100 18.51100 +H 0.89000 17.20100 19.04100 +H 1.94000 18.45100 19.09100 +O 12.80100 9.27000 6.72000 +H 13.40100 8.46000 6.74000 +H 13.20100 9.96000 6.12000 +O 8.30000 12.73100 14.22100 +H 8.25000 13.06100 15.17100 +H 7.44000 12.92100 13.76100 +O 6.72000 16.16100 1.54000 +H 6.81000 16.26100 0.55000 +H 6.32000 15.27100 1.75000 +O 16.50100 17.20100 13.94100 +H 17.03100 17.30100 13.10100 +H 16.23100 18.10100 14.28100 +O 18.41100 1.75000 9.63100 +H 18.80100 0.90000 9.27100 +H 17.43100 1.77000 9.44100 +O 2.63000 3.26000 7.20000 +H 1.84000 3.77000 6.86000 +H 2.54000 3.11000 8.18000 +O 11.94100 16.12100 0.31000 +H 12.00100 15.19100 0.68000 +H 11.35100 16.12100 -0.49000 +O 8.22000 10.02100 13.72100 +H 8.62000 10.01100 12.80100 +H 8.32000 10.94100 14.12100 +O 9.16000 9.10000 2.91000 +H 9.79000 9.48000 2.23000 +H 9.56000 8.27000 3.30000 +O 15.04100 16.07100 0.44000 +H 14.12100 16.44100 0.51000 +H 15.42100 16.27100 -0.46000 +O 3.72000 12.88100 14.90100 +H 3.59000 13.81100 14.56100 +H 2.88000 12.36100 14.77100 +O 10.39100 10.98100 6.96000 +H 9.69100 10.51100 7.50000 +H 10.98100 10.30100 6.53000 +O 10.14100 2.36000 9.71100 +H 10.06100 2.00000 8.78100 +H 10.12100 1.60000 10.36100 +O 5.90000 14.87100 4.91000 +H 6.32000 14.29100 4.21000 +H 5.46000 15.66100 4.47000 +O 17.09100 3.85000 13.81100 +H 17.82100 4.54000 13.85100 +H 17.37100 3.10000 13.22100 +O 2.55000 13.48100 2.90000 +H 1.59000 13.49100 2.63000 +H 2.67000 14.01100 3.74000 +O 6.72000 2.03000 14.89100 +H 7.62000 1.87000 14.49100 +H 6.80000 2.08000 15.88100 +O 14.40100 8.56000 13.98100 +H 13.83100 9.08000 13.35100 +H 15.36100 8.68000 13.74100 +O 0.72000 1.66000 3.18000 +H 0.55000 2.49000 2.64000 +H 1.62000 1.29000 2.96000 +O 11.83100 13.35100 1.19000 +H 10.84100 13.24100 1.21000 +H 12.17100 13.50100 2.12000 +O 6.13000 8.42000 14.31100 +H 6.69000 9.23000 14.14100 +H 6.72000 7.62000 14.34100 +O 14.93100 17.67100 9.59100 +H 15.26100 18.31100 8.90100 +H 15.59100 17.61100 10.34100 +O 7.16000 5.65000 17.08100 +H 7.35000 6.30000 17.82100 +H 7.76000 4.85000 17.17100 +O 14.50100 12.20100 16.33100 +H 14.41100 12.10100 17.32100 +H 15.46100 12.13100 16.07100 +O 3.90000 17.41100 15.60100 +H 2.99000 17.82100 15.58100 +H 3.83000 16.47100 15.92100 +O 16.74100 8.83000 12.54100 +H 16.47100 7.94000 12.17100 +H 16.75100 9.51000 11.81100 +O 12.25100 3.25000 4.49000 +H 12.90100 2.51000 4.38000 +H 12.45100 3.75000 5.33000 +O 5.94000 7.45000 6.52000 +H 6.44000 8.30000 6.33000 +H 5.06000 7.47000 6.04000 +O 17.77100 3.42000 16.42100 +H 17.60100 3.73000 15.48100 +H 16.93100 3.05000 16.80100 +O 17.30100 9.34100 15.17100 +H 17.68100 10.25100 15.32100 +H 17.22100 9.17100 14.18100 +O 8.59000 13.74100 0.16000 +H 8.13000 13.89100 1.04000 +H 9.03000 14.59100 -0.14000 +O 6.61000 17.90100 9.53100 +H 6.15000 18.78100 9.40100 +H 7.60000 18.02100 9.46100 +O 14.08100 18.51100 17.20100 +H 13.12100 18.40100 16.93100 +H 14.64100 17.84100 16.72100 +O 8.59000 9.56100 8.61000 +H 9.13000 8.87100 9.09000 +H 8.27000 10.25100 9.27000 +O 10.83100 9.84100 0.87000 +H 10.60100 10.37100 0.05000 +H 11.64100 9.28100 0.68000 +O 18.61100 15.69100 8.51000 +H 17.90100 15.57100 7.81000 +H 18.62100 14.90100 9.11000 +O 2.21000 13.14100 18.44100 +H 1.56000 12.41100 18.23100 +H 2.25000 13.28100 19.42100 +O 0.79000 12.40100 6.53000 +H 0.78000 11.93100 7.41000 +H 1.61000 12.12100 6.02000 +O 6.72000 13.91100 16.24100 +H 5.94000 13.41100 16.62100 +H 6.69000 14.86100 16.55100 +O 18.24100 1.92000 12.27100 +H 18.20100 1.02000 12.71100 +H 18.27100 1.81000 11.28100 +O 4.28000 4.24000 5.20000 +H 4.58000 3.52000 4.58000 +H 3.89000 3.84000 6.03000 +O 17.05100 14.87100 11.04100 +H 16.12100 14.62100 10.77100 +H 17.31100 14.37100 11.86100 +O 3.17000 5.47000 12.80100 +H 3.55000 4.88000 13.52100 +H 3.57000 5.21000 11.92100 +O 8.12000 15.86100 6.87000 +H 8.44000 15.96100 5.93000 +H 7.33000 15.24100 6.89000 +O 14.24100 2.14000 11.12100 +H 14.76100 1.49000 11.67100 +H 13.75100 2.77000 11.73100 +O 10.01100 0.34000 11.54100 +H 9.38100 -0.38000 11.23100 +H 10.94100 -0.02000 11.54100 +O 7.70000 13.30100 3.01000 +H 7.24000 12.43100 3.18000 +H 8.61000 13.27100 3.42000 +O 6.18000 15.67100 12.84100 +H 6.13000 16.49100 13.41100 +H 7.07000 15.64100 12.39100 +O 13.52100 0.52000 1.68000 +H 13.87100 0.11000 0.84000 +H 12.62100 0.14000 1.88000 +O 13.00100 4.53000 6.91000 +H 12.41100 5.33000 6.95000 +H 13.15100 4.18000 7.84000 +O 15.93100 2.21000 8.82000 +H 15.09100 2.20000 9.36000 +H 15.95100 3.04000 8.26000 +O 0.39000 10.77100 3.00000 +H 1.38000 10.66100 2.91000 +H -0.01000 9.91100 3.32000 +O 8.75000 16.46100 3.37000 +H 7.98000 16.11100 2.83000 +H 8.43000 17.17100 3.99000 +O 2.30000 6.28000 19.75100 +H 1.37000 6.26000 20.12100 +H 2.31000 5.89000 18.83100 +O 15.55100 15.11100 25.65100 +H 14.98100 14.95100 26.46100 +H 14.96100 15.21100 24.85100 +O 17.43100 6.18000 27.18100 +H 17.76100 7.12000 27.18100 +H 17.94100 5.64000 27.84100 +O 11.35100 7.03000 25.79100 +H 11.92100 7.81000 25.54100 +H 10.75100 7.29000 26.55100 +O 17.55100 6.07000 20.93100 +H 17.43100 5.94000 19.94100 +H 17.25100 5.26000 21.42100 +O 7.68000 11.44100 1.02100 +H 6.90000 11.61100 1.62100 +H 8.02000 12.31100 0.66100 +O 6.85000 10.12100 25.27100 +H 7.54000 9.96100 25.97100 +H 6.12000 10.69100 25.65100 +O 2.31000 17.13100 23.45100 +H 2.65000 17.90100 23.99100 +H 2.75000 17.13100 22.55100 +O 2.40000 10.91100 27.48100 +H 2.54000 10.07100 28.00100 +H 1.85000 11.55100 28.03100 +O 6.06000 9.64100 19.85100 +H 6.13000 10.48100 19.31100 +H 6.52000 9.77100 20.73100 +O 18.42100 17.67100 22.21100 +H 18.96100 17.38100 23.01100 +H 18.72100 18.57100 21.92100 +O 16.89100 9.22000 24.74100 +H 17.84100 8.93000 24.82100 +H 16.81100 9.87000 23.99100 +O 16.41100 11.08100 22.94100 +H 17.27100 11.10100 22.42100 +H 16.55100 11.55100 23.82100 +O 6.13000 13.65100 25.88100 +H 5.64000 12.78100 25.97100 +H 5.90000 14.08100 25.01100 +O 8.09000 0.04000 23.64100 +H 8.49000 0.95000 23.55100 +H 7.09000 0.12000 23.70100 +O 15.25100 9.99100 20.52100 +H 14.62100 9.23100 20.65100 +H 15.73100 10.17100 21.38100 +O 16.67100 17.14100 24.34100 +H 16.26100 16.91100 23.46100 +H 16.49100 16.40100 24.99100 +O 5.98000 7.29000 21.32100 +H 6.22000 7.98000 20.64100 +H 5.20000 7.62000 21.86100 +O 12.81100 3.45000 0.23100 +H 11.95100 2.95000 0.10100 +H 13.43100 2.91000 0.79100 +O 15.76100 16.62100 21.69100 +H 16.65100 17.08100 21.72100 +H 15.55100 16.38100 20.74100 +O 10.41100 7.01000 22.91100 +H 10.67100 6.97000 23.87100 +H 9.56100 6.50000 22.77100 +O 8.65000 3.48000 20.57100 +H 9.24000 4.11000 20.08100 +H 8.84000 2.54000 20.28100 +O 13.62100 11.44100 24.07100 +H 14.45100 11.15100 23.59100 +H 13.13100 12.11100 23.51100 +O 10.08100 14.56100 23.39100 +H 9.62100 15.28100 22.87100 +H 10.04100 14.76100 24.37100 +O 3.51000 18.01100 27.15100 +H 4.01000 17.15100 27.21100 +H 4.16000 18.78100 27.12100 +O 17.95100 10.66100 27.35100 +H 17.33100 10.51100 26.59100 +H 17.43100 10.77100 28.20100 +O 3.21000 9.43100 21.04100 +H 4.03000 9.82100 20.62100 +H 2.94000 8.61100 20.55100 +O 14.58100 7.35000 25.90100 +H 14.53100 6.70000 26.65100 +H 15.38100 7.94000 26.03100 +O 11.11100 17.76100 20.99100 +H 10.51100 17.14100 21.49100 +H 11.42100 17.32100 20.14100 +O 16.32100 13.77100 19.43100 +H 16.00100 14.71100 19.33100 +H 15.56100 13.14100 19.31100 +O 4.64000 17.43100 21.85100 +H 4.97000 17.82100 22.71100 +H 5.40000 17.36100 21.20100 +O 14.00100 1.07000 22.88100 +H 13.76100 0.70000 21.98100 +H 14.99100 1.23000 22.92100 +O 3.82000 7.00000 23.42100 +H 4.27000 6.10000 23.39100 +H 2.88000 6.89000 23.75100 +O 6.14000 1.22000 19.79100 +H 7.12000 1.00000 19.86100 +H 5.83000 1.05000 18.86100 +O 5.39000 0.64000 23.74100 +H 4.58000 0.65000 24.32100 +H 5.42000 1.47000 23.19100 +O 14.34100 11.88100 19.03100 +H 14.66100 11.12100 19.60100 +H 13.42100 12.15100 19.33100 +O 9.35100 2.36000 23.42100 +H 8.87100 2.77000 22.64100 +H 10.34100 2.34000 23.23100 +O 4.59000 11.52100 26.03100 +H 3.88000 11.25100 26.68100 +H 4.33000 11.24100 25.10100 +O 12.71100 17.97100 24.53100 +H 13.15100 18.61100 23.89100 +H 12.21100 18.49100 25.23100 +O 10.32100 5.49000 18.78100 +H 9.91100 6.31000 18.39100 +H 10.96100 5.75000 19.51100 +O 8.66000 4.54000 25.04100 +H 8.34000 5.26000 24.42100 +H 8.90000 3.73000 24.51100 +O 10.17100 0.39000 26.15100 +H 9.45100 0.44000 25.46100 +H 9.93100 -0.30000 26.84100 +O 18.57100 8.33000 22.39100 +H 18.99100 7.69000 23.03100 +H 18.19100 7.82000 21.61100 +O 4.88000 13.85100 20.36100 +H 4.01000 13.70100 20.83100 +H 4.71000 14.11100 19.41100 +O 16.64100 12.80100 25.19100 +H 17.63100 12.88100 25.33100 +H 16.19100 13.64100 25.50100 +O 13.90100 5.75000 19.40100 +H 13.36100 5.54000 20.21100 +H 14.81100 5.34000 19.49100 +O 5.27000 2.56000 21.90100 +H 5.54000 1.97000 21.15100 +H 5.27000 3.51000 21.59100 +O 7.05000 10.50100 22.30100 +H 6.91000 10.57100 23.29100 +H 7.89000 9.99100 22.12100 +O 3.67000 11.00100 23.63100 +H 3.60000 11.83100 23.07100 +H 3.71000 10.20100 23.03100 +O 5.66000 5.37000 27.27100 +H 5.78000 6.03000 26.53100 +H 6.12000 5.71000 28.10100 +O 12.52100 13.48100 22.50100 +H 13.02100 14.25100 22.90100 +H 11.57100 13.50100 22.82100 +O 15.82100 6.39000 23.34100 +H 15.51100 7.00000 24.07100 +H 16.32100 6.91000 22.65100 +O 12.09100 5.25000 21.37100 +H 12.22100 4.41000 21.91100 +H 11.80100 5.99000 21.97100 +O 16.22100 3.67000 22.36100 +H 16.24100 2.91000 23.00100 +H 15.74100 4.44000 22.76100 +O 15.40100 2.74000 19.87100 +H 14.79100 1.99000 20.10100 +H 15.62100 3.26000 20.70100 +O 13.03100 8.38000 19.04100 +H 13.37100 7.45000 19.19100 +H 13.21100 8.65000 18.09100 +O 10.68100 13.33100 27.11100 +H 10.75100 12.49100 26.56100 +H 11.30100 14.02100 26.75100 +O 8.15000 5.72000 21.87100 +H 8.22000 4.83000 21.41100 +H 7.21000 6.06000 21.79100 +O 4.73000 5.00000 20.53100 +H 5.34000 5.80000 20.57100 +H 3.78000 5.31000 20.60100 +O 13.02100 8.55000 21.71100 +H 12.16100 8.24000 22.13100 +H 12.98100 8.41000 20.72100 +O 17.26100 1.35000 23.85100 +H 17.99100 1.18000 23.18100 +H 16.95100 0.48000 24.23100 +O 13.88100 15.73100 23.39100 +H 14.55100 15.85100 22.65100 +H 13.48100 16.62100 23.62100 +O 12.80100 9.27000 25.34100 +H 13.40100 8.46000 25.36100 +H 13.20100 9.96000 24.74100 +O 6.72000 16.16100 20.16100 +H 6.81000 16.26100 19.17100 +H 6.32000 15.27100 20.37100 +O 18.41100 1.75000 0.42100 +H 18.80100 0.90000 0.06100 +H 17.43100 1.77000 0.23100 +O 2.63000 3.26000 25.82100 +H 1.84000 3.77000 25.48100 +H 2.54000 3.11000 26.80100 +O 11.94100 16.12100 18.93100 +H 12.00100 15.19100 19.30100 +H 11.35100 16.12100 18.13100 +O 9.16000 9.10000 21.53100 +H 9.79000 9.48000 20.85100 +H 9.56000 8.27000 21.92100 +O 15.04100 16.07100 19.06100 +H 14.12100 16.44100 19.13100 +H 15.42100 16.27100 18.16100 +O 10.39100 10.98100 25.58100 +H 9.69100 10.51100 26.12100 +H 10.98100 10.30100 25.15100 +O 5.90000 14.87100 23.53100 +H 6.32000 14.29100 22.83100 +H 5.46000 15.66100 23.09100 +O 2.55000 13.48100 21.52100 +H 1.59000 13.49100 21.25100 +H 2.67000 14.01100 22.36100 +O 0.75000 3.45000 18.95100 +H -0.17000 3.17000 18.66100 +H 1.06000 4.22000 18.39100 +O 0.72000 1.66000 21.80100 +H 0.55000 2.49000 21.26100 +H 1.62000 1.29000 21.58100 +O 11.83100 13.35100 19.81100 +H 10.84100 13.24100 19.83100 +H 12.17100 13.50100 20.74100 +O 12.25100 3.25000 23.11100 +H 12.90100 2.51000 23.00100 +H 12.45100 3.75000 23.95100 +O 5.94000 7.45000 25.14100 +H 6.44000 8.30000 24.95100 +H 5.06000 7.47000 24.66100 +O 8.59000 13.74100 18.78100 +H 8.13000 13.89100 19.66100 +H 9.03000 14.59100 18.48100 +O 6.61000 17.90100 0.32100 +H 6.15000 18.78100 0.19100 +H 7.60000 18.02100 0.25100 +O 8.59000 9.56100 27.23100 +H 9.13000 8.87100 27.71100 +H 8.27000 10.25100 27.89100 +O 10.83100 9.84100 19.49100 +H 10.60100 10.37100 18.67100 +H 11.64100 9.28100 19.30100 +O 18.61100 15.69100 27.13100 +H 17.90100 15.57100 26.43100 +H 18.62100 14.90100 27.73100 +O 0.79000 12.40100 25.15100 +H 0.78000 11.93100 26.03100 +H 1.61000 12.12100 24.64100 +O 4.28000 4.24000 23.82100 +H 4.58000 3.52000 23.20100 +H 3.89000 3.84000 24.65100 +O 8.12000 15.86100 25.49100 +H 8.44000 15.96100 24.55100 +H 7.33000 15.24100 25.51100 +O 7.70000 13.30100 21.63100 +H 7.24000 12.43100 21.80100 +H 8.61000 13.27100 22.04100 +O 13.52100 0.52000 20.30100 +H 13.87100 0.11000 19.46100 +H 12.62100 0.14000 20.50100 +O 13.00100 4.53000 25.53100 +H 12.41100 5.33000 25.57100 +H 13.15100 4.18000 26.46100 +O 0.39000 10.77100 21.62100 +H 1.38000 10.66100 21.53100 +H -0.01000 9.91100 21.94100 +O 8.75000 16.46100 21.99100 +H 7.98000 16.11100 21.45100 +H 8.43000 17.17100 22.61100 +O 2.30000 24.90100 1.13000 +H 1.37000 24.88100 1.50000 +H 2.31000 24.51100 0.21000 +O 17.43100 24.80100 8.56000 +H 17.76100 25.74100 8.56000 +H 17.94100 24.26100 9.22000 +O 11.35100 25.65100 7.17000 +H 11.92100 26.43100 6.92000 +H 10.75100 25.91100 7.93000 +O 17.55100 24.69100 2.31000 +H 17.43100 24.56100 1.32000 +H 17.25100 23.88100 2.80000 +O 8.50000 26.60100 18.23100 +H 8.46000 27.36100 18.88100 +H 8.72000 26.96100 17.32100 +O 6.85000 0.91100 6.65000 +H 7.54000 0.75100 7.35000 +H 6.12000 1.48100 7.03000 +O 4.38000 22.54100 14.99100 +H 5.20000 21.98100 15.08100 +H 3.57000 21.96100 15.03100 +O 15.94100 19.76100 14.80100 +H 15.76100 20.43100 14.08100 +H 15.91100 20.22100 15.69100 +O 1.13000 25.99100 15.97100 +H 2.01000 25.86100 16.42100 +H 1.00000 26.96100 15.75100 +O 8.09000 18.66100 5.02000 +H 8.49000 19.57100 4.93000 +H 7.09000 18.74100 5.08000 +O 3.17000 21.13100 18.01100 +H 3.88000 21.84100 18.07100 +H 2.29000 21.52100 18.29100 +O 5.98000 25.91100 2.70000 +H 6.22000 26.60100 2.02000 +H 5.20000 26.24100 3.24000 +O 12.81100 22.07100 9.44100 +H 11.95100 21.57100 9.31100 +H 13.43100 21.53100 10.00100 +O 8.07000 24.67100 14.65100 +H 7.60000 24.64100 15.54100 +H 7.56000 24.12100 13.99100 +O 13.94100 23.31100 16.74100 +H 13.74100 23.74100 17.62100 +H 14.72100 22.69100 16.83100 +O 9.91100 22.72100 12.42100 +H 9.14100 23.06100 12.96100 +H 9.57100 22.21100 11.63100 +O 10.41100 25.63100 4.29000 +H 10.67100 25.59100 5.25000 +H 9.56100 25.12100 4.15000 +O 8.65000 22.10100 1.95000 +H 9.24000 22.73100 1.46000 +H 8.84000 21.16100 1.66000 +O 17.19100 24.47100 18.31100 +H 16.93100 25.36100 17.95100 +H 17.17100 23.79100 17.58100 +O 5.50000 20.58100 8.85000 +H 5.45000 20.53100 9.85000 +H 5.52000 21.54100 8.56000 +O 14.58100 25.97100 7.28000 +H 14.53100 25.32100 8.03000 +H 15.38100 26.56100 7.41000 +O 2.02000 21.47100 14.98100 +H 1.22000 22.07100 14.85100 +H 1.92000 20.98100 15.84100 +O 14.00100 19.69100 4.26000 +H 13.76100 19.32100 3.36000 +H 14.99100 19.85100 4.30000 +O 3.82000 25.62100 4.80000 +H 4.27000 24.72100 4.77000 +H 2.88000 25.51100 5.13000 +O 15.47100 20.84100 17.29100 +H 15.42100 21.21100 18.21100 +H 14.75100 20.15100 17.17100 +O 7.81000 21.26100 17.49100 +H 8.48000 20.65100 17.92100 +H 7.08000 21.45100 18.14100 +O 13.51100 24.52100 14.33100 +H 13.79100 24.09100 15.18100 +H 13.76100 25.48100 14.34100 +O 9.22000 23.65100 8.99000 +H 8.97000 23.56100 8.03000 +H 9.70000 22.83100 9.30000 +O 5.39000 19.26100 5.12000 +H 4.58000 19.27100 5.70000 +H 5.42000 20.09100 4.57000 +O 2.97000 18.97100 1.71000 +H 3.46000 19.81100 1.50000 +H 3.59000 18.32100 2.16000 +O 9.35100 20.98100 4.80000 +H 8.87100 21.39100 4.02000 +H 10.34100 20.96100 4.61000 +O 10.76100 25.45100 14.64100 +H 9.96100 24.84100 14.67100 +H 11.57100 24.92100 14.40100 +O 4.59000 2.31100 7.41000 +H 3.88000 2.04100 8.06000 +H 4.33000 2.03100 6.48000 +O 10.32100 24.11100 0.16000 +H 9.91100 24.93100 -0.23000 +H 10.96100 24.37100 0.89000 +O 0.78000 24.18100 13.86100 +H 1.70000 24.17100 13.45100 +H 0.72000 24.92100 14.53100 +O 5.61000 20.84100 11.47100 +H 5.99000 20.00100 11.84100 +H 4.73000 21.03100 11.91100 +O 8.66000 23.16100 6.42000 +H 8.34000 23.88100 5.80000 +H 8.90000 22.35100 5.89000 +O 10.17100 19.01100 7.53000 +H 9.45100 19.06100 6.84000 +H 9.93100 18.32100 8.22000 +O 14.66100 24.52100 9.92100 +H 14.36100 23.57100 9.99100 +H 15.39100 24.68100 10.58100 +O 18.57100 26.95100 3.77000 +H 18.99100 26.31100 4.41000 +H 18.19100 26.44100 2.99000 +O 13.90100 24.37100 0.78000 +H 13.36100 24.16100 1.59000 +H 14.81100 23.96100 0.87000 +O 5.27000 21.18100 3.28000 +H 5.54000 20.59100 2.53000 +H 5.27000 22.13100 2.97000 +O 9.15000 19.51100 14.02100 +H 9.40000 19.31100 13.07100 +H 9.87000 20.07100 14.44100 +O 4.10000 26.75100 12.51100 +H 4.96000 26.87100 13.01100 +H 3.68000 25.88100 12.78100 +O 12.74100 22.48100 12.62100 +H 12.95100 23.22100 13.26100 +H 11.85100 22.65100 12.19100 +O 5.66000 23.99100 8.65000 +H 5.78000 24.65100 7.91000 +H 6.12000 24.33100 9.48000 +O 15.82100 25.01100 4.72000 +H 15.51100 25.62100 5.45000 +H 16.32100 25.53100 4.03000 +O 4.02000 26.13100 15.98100 +H 4.70000 26.68100 15.51100 +H 4.42000 25.25100 16.25100 +O 15.87100 26.41100 16.70100 +H 14.95100 26.79100 16.65100 +H 16.47100 26.88100 16.05100 +O 10.13100 19.67100 17.70100 +H 10.19100 20.52100 17.18100 +H 10.45100 18.91100 17.13100 +O 5.04000 19.12100 17.40100 +H 4.62000 18.55100 16.70100 +H 4.38000 19.81100 17.72100 +O 5.73000 27.32100 10.29100 +H 6.17000 28.21100 10.20100 +H 5.10000 27.32100 11.06100 +O 13.60100 27.24100 10.45100 +H 12.85100 27.24100 9.79100 +H 13.97100 26.32100 10.54100 +O 12.09100 23.87100 2.75000 +H 12.22100 23.03100 3.29000 +H 11.80100 24.61100 3.35000 +O 3.07000 20.75100 12.31100 +H 2.84000 21.12100 13.21100 +H 2.77000 19.80100 12.25100 +O 7.32000 24.96100 10.64100 +H 7.91000 24.70100 9.88100 +H 7.04000 25.92100 10.53100 +O 17.28100 0.14100 18.54100 +H 16.82100 0.07100 17.65100 +H 16.66100 0.58100 19.20100 +O 3.07000 19.25100 6.18000 +H 2.96000 20.19100 6.51000 +H 3.02000 18.62100 6.95000 +O 16.22100 22.29100 3.74000 +H 16.24100 21.53100 4.38000 +H 15.74100 23.06100 4.14000 +O 10.23100 26.28100 9.66100 +H 10.38100 26.49100 10.62100 +H 9.93100 25.33100 9.57100 +O 15.40100 21.36100 1.25000 +H 14.79100 20.61100 1.48000 +H 15.62100 21.88100 2.08000 +O 3.19000 26.72100 9.49100 +H 4.12000 27.08100 9.54100 +H 3.13000 25.87100 10.01100 +O 3.39000 23.71100 10.06100 +H 2.87000 22.88100 9.89100 +H 4.16000 23.76100 9.42100 +O 11.38100 22.42100 16.78100 +H 10.93100 23.05100 17.42100 +H 12.31100 22.73100 16.61100 +O 11.60100 20.69100 14.77100 +H 11.60100 21.33100 15.54100 +H 11.88100 21.17100 13.94100 +O 10.02100 26.58100 12.38100 +H 10.43100 26.26100 13.24100 +H 9.06100 26.31100 12.35100 +O 0.40000 24.06100 11.14100 +H 1.25000 23.73100 10.73100 +H 0.53000 24.21100 12.12100 +O 1.89000 23.82100 17.22100 +H 2.48000 23.42100 16.52100 +H 1.31000 24.53100 16.81100 +O 8.15000 24.34100 3.25000 +H 8.22000 23.45100 2.79000 +H 7.21000 24.68100 3.17000 +O 16.57100 24.66100 12.06100 +H 16.19100 23.97100 12.68100 +H 17.39100 24.30100 11.62100 +O 6.71000 23.26100 12.69100 +H 6.37000 22.37100 12.39100 +H 6.97000 23.80100 11.89100 +O 4.73000 23.62100 1.91000 +H 5.34000 24.42100 1.95000 +H 3.78000 23.93100 1.98000 +O 9.03000 19.48100 1.33000 +H 9.54000 19.49100 0.47000 +H 9.59000 19.06100 2.04000 +O 17.26100 19.97100 5.23000 +H 17.99100 19.80100 4.56000 +H 16.95100 19.10100 5.61000 +O 12.80100 0.06100 6.72000 +H 13.40100 -0.74900 6.74000 +H 13.20100 0.75100 6.12000 +O 18.41100 20.37100 9.63100 +H 18.80100 19.52100 9.27100 +H 17.43100 20.39100 9.44100 +O 2.63000 21.88100 7.20000 +H 1.84000 22.39100 6.86000 +H 2.54000 21.73100 8.18000 +O 10.14100 20.98100 9.71100 +H 10.06100 20.62100 8.78100 +H 10.12100 20.22100 10.36100 +O 17.09100 22.47100 13.81100 +H 17.82100 23.16100 13.85100 +H 17.37100 21.72100 13.22100 +O 6.72000 20.65100 14.89100 +H 7.62000 20.49100 14.49100 +H 6.80000 20.70100 15.88100 +O 14.40100 27.18100 13.98100 +H 13.83100 27.70100 13.35100 +H 15.36100 27.30100 13.74100 +O 0.72000 20.28100 3.18000 +H 0.55000 21.11100 2.64000 +H 1.62000 19.91100 2.96000 +O 6.13000 27.04100 14.31100 +H 6.69000 27.85100 14.14100 +H 6.72000 26.24100 14.34100 +O 7.16000 24.27100 17.08100 +H 7.35000 24.92100 17.82100 +H 7.76000 23.47100 17.17100 +O 16.74100 27.45100 12.54100 +H 16.47100 26.56100 12.17100 +H 16.75100 28.13100 11.81100 +O 12.25100 21.87100 4.49000 +H 12.90100 21.13100 4.38000 +H 12.45100 22.37100 5.33000 +O 5.94000 26.07100 6.52000 +H 6.44000 26.92100 6.33000 +H 5.06000 26.09100 6.04000 +O 17.77100 22.04100 16.42100 +H 17.60100 22.35100 15.48100 +H 16.93100 21.67100 16.80100 +O 17.30100 0.13100 15.17100 +H 17.68100 1.04100 15.32100 +H 17.22100 -0.03900 14.18100 +O 18.24100 20.54100 12.27100 +H 18.20100 19.64100 12.71100 +H 18.27100 20.43100 11.28100 +O 4.28000 22.86100 5.20000 +H 4.58000 22.14100 4.58000 +H 3.89000 22.46100 6.03000 +O 3.17000 24.09100 12.80100 +H 3.55000 23.50100 13.52100 +H 3.57000 23.83100 11.92100 +O 14.24100 20.76100 11.12100 +H 14.76100 20.11100 11.67100 +H 13.75100 21.39100 11.73100 +O 10.01100 18.96100 11.54100 +H 9.38100 18.24100 11.23100 +H 10.94100 18.60100 11.54100 +O 13.52100 19.14100 1.68000 +H 13.87100 18.73100 0.84000 +H 12.62100 18.76100 1.88000 +O 13.00100 23.15100 6.91000 +H 12.41100 23.95100 6.95000 +H 13.15100 22.80100 7.84000 +O 15.93100 20.83100 8.82000 +H 15.09100 20.82100 9.36000 +H 15.95100 21.66100 8.26000 +O 2.30000 24.90100 19.75100 +H 1.37000 24.88100 20.12100 +H 2.31000 24.51100 18.83100 +O 17.43100 24.80100 27.18100 +H 17.76100 25.74100 27.18100 +H 17.94100 24.26100 27.84100 +O 11.35100 25.65100 25.79100 +H 11.92100 26.43100 25.54100 +H 10.75100 25.91100 26.55100 +O 17.55100 24.69100 20.93100 +H 17.43100 24.56100 19.94100 +H 17.25100 23.88100 21.42100 +O 6.85000 0.91100 25.27100 +H 7.54000 0.75100 25.97100 +H 6.12000 1.48100 25.65100 +O 2.40000 1.70100 27.48100 +H 2.54000 0.86100 28.00100 +H 1.85000 2.34100 28.03100 +O 8.09000 18.66100 23.64100 +H 8.49000 19.57100 23.55100 +H 7.09000 18.74100 23.70100 +O 5.98000 25.91100 21.32100 +H 6.22000 26.60100 20.64100 +H 5.20000 26.24100 21.86100 +O 12.81100 22.07100 0.23100 +H 11.95100 21.57100 0.10100 +H 13.43100 21.53100 0.79100 +O 10.41100 25.63100 22.91100 +H 10.67100 25.59100 23.87100 +H 9.56100 25.12100 22.77100 +O 0.76000 26.73100 26.51100 +H 1.75000 26.61100 26.60100 +H 0.52000 27.68100 26.72100 +O 8.65000 22.10100 20.57100 +H 9.24000 22.73100 20.08100 +H 8.84000 21.16100 20.28100 +O 5.50000 20.58100 27.47100 +H 5.45000 20.53100 28.47100 +H 5.52000 21.54100 27.18100 +O 14.58100 25.97100 25.90100 +H 14.53100 25.32100 26.65100 +H 15.38100 26.56100 26.03100 +O 14.00100 19.69100 22.88100 +H 13.76100 19.32100 21.98100 +H 14.99100 19.85100 22.92100 +O 3.82000 25.62100 23.42100 +H 4.27000 24.72100 23.39100 +H 2.88000 25.51100 23.75100 +O 6.14000 19.84100 19.79100 +H 7.12000 19.62100 19.86100 +H 5.83000 19.67100 18.86100 +O 5.39000 19.26100 23.74100 +H 4.58000 19.27100 24.32100 +H 5.42000 20.09100 23.19100 +O 2.97000 18.97100 20.33100 +H 3.46000 19.81100 20.12100 +H 3.59000 18.32100 20.78100 +O 9.35100 20.98100 23.42100 +H 8.87100 21.39100 22.64100 +H 10.34100 20.96100 23.23100 +O 4.59000 2.31100 26.03100 +H 3.88000 2.04100 26.68100 +H 4.33000 2.03100 25.10100 +O 10.32100 24.11100 18.78100 +H 9.91100 24.93100 18.39100 +H 10.96100 24.37100 19.51100 +O 8.66000 23.16100 25.04100 +H 8.34000 23.88100 24.42100 +H 8.90000 22.35100 24.51100 +O 10.17100 19.01100 26.15100 +H 9.45100 19.06100 25.46100 +H 9.93100 18.32100 26.84100 +O 18.57100 26.95100 22.39100 +H 18.99100 26.31100 23.03100 +H 18.19100 26.44100 21.61100 +O 13.90100 24.37100 19.40100 +H 13.36100 24.16100 20.21100 +H 14.81100 23.96100 19.49100 +O 5.27000 21.18100 21.90100 +H 5.54000 20.59100 21.15100 +H 5.27000 22.13100 21.59100 +O 5.66000 23.99100 27.27100 +H 5.78000 24.65100 26.53100 +H 6.12000 24.33100 28.10100 +O 15.82100 25.01100 23.34100 +H 15.51100 25.62100 24.07100 +H 16.32100 25.53100 22.65100 +O 12.09100 23.87100 21.37100 +H 12.22100 23.03100 21.91100 +H 11.80100 24.61100 21.97100 +O 3.07000 19.25100 24.80100 +H 2.96000 20.19100 25.13100 +H 3.02000 18.62100 25.57100 +O 16.22100 22.29100 22.36100 +H 16.24100 21.53100 23.00100 +H 15.74100 23.06100 22.76100 +O 15.40100 21.36100 19.87100 +H 14.79100 20.61100 20.10100 +H 15.62100 21.88100 20.70100 +O 8.15000 24.34100 21.87100 +H 8.22000 23.45100 21.41100 +H 7.21000 24.68100 21.79100 +O 4.73000 23.62100 20.53100 +H 5.34000 24.42100 20.57100 +H 3.78000 23.93100 20.60100 +O 9.03000 19.48100 19.95100 +H 9.54000 19.49100 19.09100 +H 9.59000 19.06100 20.66100 +O 17.26100 19.97100 23.85100 +H 17.99100 19.80100 23.18100 +H 16.95100 19.10100 24.23100 +O 12.80100 0.06100 25.34100 +H 13.40100 -0.74900 25.36100 +H 13.20100 0.75100 24.74100 +O 18.41100 20.37100 0.42100 +H 18.80100 19.52100 0.06100 +H 17.43100 20.39100 0.23100 +O 2.63000 21.88100 25.82100 +H 1.84000 22.39100 25.48100 +H 2.54000 21.73100 26.80100 +O 9.16000 27.72100 21.53100 +H 9.79000 28.10100 20.85100 +H 9.56000 26.89100 21.92100 +O 0.75000 22.07100 18.95100 +H -0.17000 21.79100 18.66100 +H 1.06000 22.84100 18.39100 +O 0.72000 20.28100 21.80100 +H 0.55000 21.11100 21.26100 +H 1.62000 19.91100 21.58100 +O 12.25100 21.87100 23.11100 +H 12.90100 21.13100 23.00100 +H 12.45100 22.37100 23.95100 +O 5.94000 26.07100 25.14100 +H 6.44000 26.92100 24.95100 +H 5.06000 26.09100 24.66100 +O 4.28000 22.86100 23.82100 +H 4.58000 22.14100 23.20100 +H 3.89000 22.46100 24.65100 +O 13.52100 19.14100 20.30100 +H 13.87100 18.73100 19.46100 +H 12.62100 18.76100 20.50100 +O 13.00100 23.15100 25.53100 +H 12.41100 23.95100 25.57100 +H 13.15100 22.80100 26.46100 +O 20.92100 6.28000 1.13000 +H 19.99100 6.26000 1.50000 +H 20.93100 5.89000 0.21000 +O 20.87100 2.75000 9.96100 +H 21.22100 2.58000 10.88100 +H 19.99100 2.30000 9.84100 +O 18.81100 3.68000 6.47000 +H 17.99100 4.11000 6.86000 +H 18.53100 2.95000 5.84000 +O 24.31100 12.75100 11.65100 +H 23.38100 12.68100 11.28100 +H 24.42100 13.64100 12.09100 +O 27.12100 7.98000 18.23100 +H 27.08100 8.74000 18.88100 +H 27.34100 8.34000 17.32100 +O 25.47100 10.12100 6.65000 +H 26.16100 9.96100 7.35000 +H 24.74100 10.69100 7.03000 +O 25.48100 11.61100 18.03100 +H 26.08100 12.40100 18.17100 +H 24.62100 11.92100 17.62100 +O 21.97100 14.35100 10.61100 +H 21.19100 14.04100 10.08100 +H 22.55100 14.93100 10.04100 +O 23.00100 3.92000 14.99100 +H 23.82100 3.36000 15.08100 +H 22.19100 3.34000 15.03100 +O 20.93100 17.13100 4.83000 +H 21.27100 17.90100 5.37000 +H 21.37100 17.13100 3.93000 +O 20.92100 14.34100 5.38000 +H 20.66100 15.30100 5.38000 +H 20.21100 13.80100 5.83000 +O 21.02100 10.91100 8.86000 +H 21.16100 10.07100 9.38000 +H 20.47100 11.55100 9.41000 +O 24.82100 17.86100 14.39100 +H 23.90100 17.69100 14.74100 +H 25.10100 18.78100 14.65100 +O 24.68100 9.64100 1.23000 +H 24.75100 10.48100 0.69000 +H 25.14100 9.77100 2.11000 +O 19.84100 6.43000 5.63000 +H 19.39100 5.55000 5.80000 +H 19.83100 6.97000 6.47000 +O 18.89100 15.96100 1.17000 +H 18.70100 15.00100 1.38000 +H 18.56100 16.54100 1.92000 +O 19.75100 7.37000 15.97100 +H 20.63100 7.24000 16.42100 +H 19.62100 8.34000 15.75100 +O 24.75100 13.65100 7.26000 +H 24.26100 12.78100 7.35000 +H 24.52100 14.08100 6.39000 +O 26.71100 0.04000 5.02000 +H 27.11100 0.95000 4.93000 +H 25.71100 0.12000 5.08000 +O 20.59100 9.76100 12.64100 +H 21.48100 9.31100 12.50100 +H 19.86100 9.11100 12.45100 +O 21.79100 2.51000 18.01100 +H 22.50100 3.22000 18.07100 +H 20.91100 2.90000 18.29100 +O 24.60100 7.29000 2.70000 +H 24.84100 7.98000 2.02000 +H 23.82100 7.62000 3.24000 +O 26.69100 6.05000 14.65100 +H 26.22100 6.02000 15.54100 +H 26.18100 5.50000 13.99100 +O 0.52100 8.90000 15.72100 +H 0.98100 8.06000 15.43100 +H -0.03900 9.24000 14.97100 +O 1.20100 7.01000 4.29000 +H 1.46100 6.97000 5.25000 +H 0.35100 6.50000 4.15000 +O 19.38100 8.11000 7.89000 +H 20.37100 7.99000 7.98000 +H 19.14100 9.06000 8.10000 +O 19.92100 18.21100 15.71100 +H 19.82100 18.06100 16.70100 +H 19.06100 18.57100 15.35100 +O 24.12100 1.96000 8.85000 +H 24.07100 1.91000 9.85000 +H 24.14100 2.92000 8.56000 +O 0.87100 14.56100 4.77000 +H 0.41100 15.28100 4.25000 +H 0.83100 14.76100 5.75000 +O 22.13100 18.01100 8.53000 +H 22.63100 17.15100 8.59000 +H 22.78100 18.78100 8.50000 +O 21.83100 9.43100 2.42000 +H 22.65100 9.82100 2.00000 +H 21.56100 8.61100 1.93000 +O 23.23100 12.66100 17.27100 +H 22.73100 12.67100 16.41100 +H 22.60100 12.48100 18.03100 +O 1.90100 17.76100 2.37000 +H 1.30100 17.14100 2.87000 +H 2.21100 17.32100 1.52000 +O 20.64100 2.85000 14.98100 +H 19.84100 3.45000 14.85100 +H 20.54100 2.36000 15.84100 +O 23.26100 17.43100 3.23000 +H 23.59100 17.82100 4.09000 +H 24.02100 17.36100 2.58000 +O 21.11100 17.85100 12.41100 +H 21.68100 17.20100 12.91100 +H 20.95100 17.52100 11.48100 +O 22.44100 7.00000 4.80000 +H 22.89100 6.10000 4.77000 +H 21.50100 6.89000 5.13000 +O 24.76100 1.22000 1.17000 +H 25.74100 1.00000 1.24000 +H 24.45100 1.05000 0.24000 +O 26.43100 2.64000 17.49100 +H 27.10100 2.03000 17.92100 +H 25.70100 2.83000 18.14100 +O 26.65100 14.02100 9.24000 +H 27.55100 14.16100 8.82000 +H 25.94100 14.04100 8.53000 +O 0.01100 5.03000 8.99000 +H -0.23900 4.94000 8.03000 +H 0.49100 4.21000 9.30000 +O 24.01100 0.64000 5.12000 +H 23.20100 0.65000 5.70000 +H 24.04100 1.47000 4.57000 +O 21.59100 0.35000 1.71000 +H 22.08100 1.19000 1.50000 +H 22.21100 -0.30000 2.16000 +O 0.14100 2.36000 4.80000 +H -0.33900 2.77000 4.02000 +H 1.13100 2.34000 4.61000 +O 23.21100 11.52100 7.41000 +H 22.50100 11.25100 8.06000 +H 22.95100 11.24100 6.48000 +O 19.40100 5.56000 13.86100 +H 20.32100 5.55000 13.45100 +H 19.34100 6.30000 14.53100 +O 24.23100 2.22000 11.47100 +H 24.61100 1.38000 11.84100 +H 23.35100 2.41000 11.91100 +O 27.28100 4.54000 6.42000 +H 26.96100 5.26000 5.80000 +H 27.52100 3.73000 5.89000 +O 23.50100 13.85100 1.74000 +H 22.63100 13.70100 2.21000 +H 23.33100 14.11100 0.79000 +O 23.89100 2.56000 3.28000 +H 24.16100 1.97000 2.53000 +H 23.89100 3.51000 2.97000 +O 27.77100 0.89000 14.02100 +H 28.02100 0.69000 13.07100 +H 28.49100 1.45000 14.44100 +O 22.72100 8.13000 12.51100 +H 23.58100 8.25000 13.01100 +H 22.30100 7.26000 12.78100 +O 19.26100 15.64100 13.31100 +H 18.80100 16.46100 12.97100 +H 20.24100 15.83100 13.40100 +O 22.29100 11.00100 5.01000 +H 22.22100 11.83100 4.45000 +H 22.33100 10.20100 4.41000 +O 24.28100 5.37000 8.65000 +H 24.40100 6.03000 7.91000 +H 24.74100 5.71000 9.48000 +O 22.16100 15.10100 13.29100 +H 21.95100 14.66100 12.42100 +H 23.13100 15.36100 13.32100 +O 22.64100 7.51000 15.98100 +H 23.32100 8.06000 15.51100 +H 23.04100 6.63000 16.25100 +O 23.66100 0.50000 17.40100 +H 23.24100 -0.07000 16.70100 +H 23.00100 1.19000 17.72100 +O 24.35100 8.70000 10.29100 +H 24.79100 9.59000 10.20100 +H 23.72100 8.70000 11.06100 +O 21.69100 2.13000 12.31100 +H 21.46100 2.50000 13.21100 +H 21.39100 1.18000 12.25100 +O 18.99100 13.10100 12.82100 +H 19.52100 12.61100 13.50100 +H 19.21100 14.08100 12.87100 +O 25.94100 6.34000 10.64100 +H 26.53100 6.08000 9.88100 +H 25.66100 7.30000 10.53100 +O 21.69100 0.63000 6.18000 +H 21.58100 1.57000 6.51000 +H 21.64100 0.00000 6.95000 +O 1.02100 7.66000 9.66100 +H 1.17100 7.87000 10.62100 +H 0.72100 6.71000 9.57100 +O 0.59100 15.73100 17.00100 +H 0.39100 16.17100 16.12100 +H 0.98100 14.82100 16.84100 +O 22.12100 8.98000 18.04100 +H 22.88100 9.42000 18.52100 +H 22.47100 8.51000 17.22100 +O 21.81100 8.10000 9.49100 +H 22.74100 8.46000 9.54100 +H 21.75100 7.25000 10.01100 +O 22.01100 5.09000 10.06100 +H 21.49100 4.26000 9.89100 +H 22.78100 5.14000 9.42100 +O 18.70100 13.26100 2.00000 +H 17.77100 13.47100 1.69000 +H 18.80100 12.27100 2.13000 +O 19.50100 18.01100 9.27000 +H 19.08100 17.15100 9.00000 +H 20.44100 18.04100 8.93000 +O 23.66100 15.68100 9.10000 +H 24.32100 16.42100 9.19000 +H 24.10100 14.89100 8.68000 +O 0.81100 7.96000 12.38100 +H 1.22100 7.64000 13.24100 +H -0.14900 7.69000 12.35100 +O 19.02100 5.44000 11.14100 +H 19.87100 5.11000 10.73100 +H 19.15100 5.59000 12.12100 +O 20.51100 5.20000 17.22100 +H 21.10100 4.80000 16.52100 +H 19.93100 5.91000 16.81100 +O 26.77100 5.72000 3.25000 +H 26.84100 4.83000 2.79000 +H 25.83100 6.06000 3.17000 +O 21.14100 15.64100 17.44100 +H 20.84100 16.21100 18.20100 +H 21.07100 14.67100 17.70100 +O 25.33100 4.64000 12.69100 +H 24.99100 3.75000 12.39100 +H 25.59100 5.18000 11.89100 +O 0.09100 16.78100 14.65100 +H -0.14900 16.60100 13.70100 +H 0.39100 17.72100 14.75100 +O 23.35100 5.00000 1.91000 +H 23.96100 5.80000 1.95000 +H 22.40100 5.31000 1.98000 +O 20.21100 11.37100 14.66100 +H 20.43100 10.76100 15.42100 +H 20.31100 10.88100 13.80100 +O 19.45100 12.58100 10.22100 +H 19.40100 12.57100 11.22100 +H 18.62100 12.17100 9.84100 +O 25.50100 16.62100 17.16100 +H 24.94100 17.43100 17.25100 +H 26.02100 16.66100 16.30100 +O 27.65100 0.86000 1.33000 +H 28.16100 0.87000 0.47000 +H 28.21100 0.44000 2.04000 +O 19.92100 17.94100 18.51100 +H 19.51100 17.20100 19.04100 +H 20.56100 18.45100 19.09100 +O 26.92100 12.73100 14.22100 +H 26.87100 13.06100 15.17100 +H 26.06100 12.92100 13.76100 +O 25.34100 16.16100 1.54000 +H 25.43100 16.26100 0.55000 +H 24.94100 15.27100 1.75000 +O 21.25100 3.26000 7.20000 +H 20.46100 3.77000 6.86000 +H 21.16100 3.11000 8.18000 +O 26.84100 10.02100 13.72100 +H 27.24100 10.01100 12.80100 +H 26.94100 10.94100 14.12100 +O 22.34100 12.88100 14.90100 +H 22.21100 13.81100 14.56100 +H 21.50100 12.36100 14.77100 +O 0.93100 2.36000 9.71100 +H 0.85100 2.00000 8.78100 +H 0.91100 1.60000 10.36100 +O 24.52100 14.87100 4.91000 +H 24.94100 14.29100 4.21000 +H 24.08100 15.66100 4.47000 +O 21.17100 13.48100 2.90000 +H 20.21100 13.49100 2.63000 +H 21.29100 14.01100 3.74000 +O 19.67100 10.13100 17.26100 +H 18.90100 9.80100 17.80100 +H 20.52100 9.83100 17.68100 +O 25.34100 2.03000 14.89100 +H 26.24100 1.87000 14.49100 +H 25.42100 2.08000 15.88100 +O 19.34100 1.66000 3.18000 +H 19.17100 2.49000 2.64000 +H 20.24100 1.29000 2.96000 +O 24.75100 8.42000 14.31100 +H 25.31100 9.23000 14.14100 +H 25.34100 7.62000 14.34100 +O 25.78100 5.65000 17.08100 +H 25.97100 6.30000 17.82100 +H 26.38100 4.85000 17.17100 +O 22.52100 17.41100 15.60100 +H 21.61100 17.82100 15.58100 +H 22.45100 16.47100 15.92100 +O 24.56100 7.45000 6.52000 +H 25.06100 8.30000 6.33000 +H 23.68100 7.47000 6.04000 +O 27.21100 13.74100 0.16000 +H 26.75100 13.89100 1.04000 +H 27.65100 14.59100 -0.14000 +O 25.23100 17.90100 9.53100 +H 24.77100 18.78100 9.40100 +H 26.22100 18.02100 9.46100 +O 27.21100 9.56100 8.61000 +H 27.75100 8.87100 9.09000 +H 26.89100 10.25100 9.27000 +O 20.83100 13.14100 18.44100 +H 20.18100 12.41100 18.23100 +H 20.87100 13.28100 19.42100 +O 19.41100 12.40100 6.53000 +H 19.40100 11.93100 7.41000 +H 20.23100 12.12100 6.02000 +O 25.34100 13.91100 16.24100 +H 24.56100 13.41100 16.62100 +H 25.31100 14.86100 16.55100 +O 22.90100 4.24000 5.20000 +H 23.20100 3.52000 4.58000 +H 22.51100 3.84000 6.03000 +O 21.79100 5.47000 12.80100 +H 22.17100 4.88000 13.52100 +H 22.19100 5.21000 11.92100 +O 26.74100 15.86100 6.87000 +H 27.06100 15.96100 5.93000 +H 25.95100 15.24100 6.89000 +O 0.80100 0.34000 11.54100 +H 0.17100 -0.38000 11.23100 +H 1.73100 -0.02000 11.54100 +O 26.32100 13.30100 3.01000 +H 25.86100 12.43100 3.18000 +H 27.23100 13.27100 3.42000 +O 24.80100 15.67100 12.84100 +H 24.75100 16.49100 13.41100 +H 25.69100 15.64100 12.39100 +O 19.01100 10.77100 3.00000 +H 20.00100 10.66100 2.91000 +H 18.61100 9.91100 3.32000 +O 27.37100 16.46100 3.37000 +H 26.60100 16.11100 2.83000 +H 27.05100 17.17100 3.99000 +O 20.92100 6.28000 19.75100 +H 19.99100 6.26000 20.12100 +H 20.93100 5.89000 18.83100 +O 20.87100 2.75000 0.75100 +H 21.22100 2.58000 1.67100 +H 19.99100 2.30000 0.63100 +O 18.81100 3.68000 25.09100 +H 17.99100 4.11000 25.48100 +H 18.53100 2.95000 24.46100 +O 26.30100 11.44100 1.02100 +H 25.52100 11.61100 1.62100 +H 26.64100 12.31100 0.66100 +O 25.47100 10.12100 25.27100 +H 26.16100 9.96100 25.97100 +H 24.74100 10.69100 25.65100 +O 20.93100 17.13100 23.45100 +H 21.27100 17.90100 23.99100 +H 21.37100 17.13100 22.55100 +O 20.92100 14.34100 24.00100 +H 20.66100 15.30100 24.00100 +H 20.21100 13.80100 24.45100 +O 21.02100 10.91100 27.48100 +H 21.16100 10.07100 28.00100 +H 20.47100 11.55100 28.03100 +O 24.68100 9.64100 19.85100 +H 24.75100 10.48100 19.31100 +H 25.14100 9.77100 20.73100 +O 19.84100 6.43000 24.25100 +H 19.39100 5.55000 24.42100 +H 19.83100 6.97000 25.09100 +O 18.89100 15.96100 19.79100 +H 18.70100 15.00100 20.00100 +H 18.56100 16.54100 20.54100 +O 24.75100 13.65100 25.88100 +H 24.26100 12.78100 25.97100 +H 24.52100 14.08100 25.01100 +O 26.71100 0.04000 23.64100 +H 27.11100 0.95000 23.55100 +H 25.71100 0.12000 23.70100 +O 24.60100 7.29000 21.32100 +H 24.84100 7.98000 20.64100 +H 23.82100 7.62000 21.86100 +O 1.20100 7.01000 22.91100 +H 1.46100 6.97000 23.87100 +H 0.35100 6.50000 22.77100 +O 19.38100 8.11000 26.51100 +H 20.37100 7.99000 26.60100 +H 19.14100 9.06000 26.72100 +O 0.87100 14.56100 23.39100 +H 0.41100 15.28100 22.87100 +H 0.83100 14.76100 24.37100 +O 22.13100 18.01100 27.15100 +H 22.63100 17.15100 27.21100 +H 22.78100 18.78100 27.12100 +O 21.83100 9.43100 21.04100 +H 22.65100 9.82100 20.62100 +H 21.56100 8.61100 20.55100 +O 1.90100 17.76100 20.99100 +H 1.30100 17.14100 21.49100 +H 2.21100 17.32100 20.14100 +O 23.26100 17.43100 21.85100 +H 23.59100 17.82100 22.71100 +H 24.02100 17.36100 21.20100 +O 22.44100 7.00000 23.42100 +H 22.89100 6.10000 23.39100 +H 21.50100 6.89000 23.75100 +O 24.76100 1.22000 19.79100 +H 25.74100 1.00000 19.86100 +H 24.45100 1.05000 18.86100 +O 0.01100 5.03000 27.61100 +H -0.23900 4.94000 26.65100 +H 0.49100 4.21000 27.92100 +O 24.01100 0.64000 23.74100 +H 23.20100 0.65000 24.32100 +H 24.04100 1.47000 23.19100 +O 21.59100 0.35000 20.33100 +H 22.08100 1.19000 20.12100 +H 22.21100 -0.30000 20.78100 +O 0.14100 2.36000 23.42100 +H -0.33900 2.77000 22.64100 +H 1.13100 2.34000 23.23100 +O 23.21100 11.52100 26.03100 +H 22.50100 11.25100 26.68100 +H 22.95100 11.24100 25.10100 +O 27.28100 4.54000 25.04100 +H 26.96100 5.26000 24.42100 +H 27.52100 3.73000 24.51100 +O 23.50100 13.85100 20.36100 +H 22.63100 13.70100 20.83100 +H 23.33100 14.11100 19.41100 +O 23.89100 2.56000 21.90100 +H 24.16100 1.97000 21.15100 +H 23.89100 3.51000 21.59100 +O 22.29100 11.00100 23.63100 +H 22.22100 11.83100 23.07100 +H 22.33100 10.20100 23.03100 +O 24.28100 5.37000 27.27100 +H 24.40100 6.03000 26.53100 +H 24.74100 5.71000 28.10100 +O 21.69100 0.63000 24.80100 +H 21.58100 1.57000 25.13100 +H 21.64100 0.00000 25.57100 +O 1.02100 7.66000 0.45100 +H 1.17100 7.87000 1.41100 +H 0.72100 6.71000 0.36100 +O 18.70100 13.26100 20.62100 +H 17.77100 13.47100 20.31100 +H 18.80100 12.27100 20.75100 +O 19.50100 18.01100 0.06100 +H 19.08100 17.15100 -0.20900 +H 20.44100 18.04100 -0.27900 +O 26.77100 5.72000 21.87100 +H 26.84100 4.83000 21.41100 +H 25.83100 6.06000 21.79100 +O 23.35100 5.00000 20.53100 +H 23.96100 5.80000 20.57100 +H 22.40100 5.31000 20.60100 +O 25.34100 16.16100 20.16100 +H 25.43100 16.26100 19.17100 +H 24.94100 15.27100 20.37100 +O 21.25100 3.26000 25.82100 +H 20.46100 3.77000 25.48100 +H 21.16100 3.11000 26.80100 +O 24.52100 14.87100 23.53100 +H 24.94100 14.29100 22.83100 +H 24.08100 15.66100 23.09100 +O 21.17100 13.48100 21.52100 +H 20.21100 13.49100 21.25100 +H 21.29100 14.01100 22.36100 +O 19.37100 3.45000 18.95100 +H 18.45100 3.17000 18.66100 +H 19.68100 4.22000 18.39100 +O 19.34100 1.66000 21.80100 +H 19.17100 2.49000 21.26100 +H 20.24100 1.29000 21.58100 +O 24.56100 7.45000 25.14100 +H 25.06100 8.30000 24.95100 +H 23.68100 7.47000 24.66100 +O 27.21100 13.74100 18.78100 +H 26.75100 13.89100 19.66100 +H 27.65100 14.59100 18.48100 +O 25.23100 17.90100 0.32100 +H 24.77100 18.78100 0.19100 +H 26.22100 18.02100 0.25100 +O 27.21100 9.56100 27.23100 +H 27.75100 8.87100 27.71100 +H 26.89100 10.25100 27.89100 +O 1.62100 9.84100 19.49100 +H 1.39100 10.37100 18.67100 +H 2.43100 9.28100 19.30100 +O 19.41100 12.40100 25.15100 +H 19.40100 11.93100 26.03100 +H 20.23100 12.12100 24.64100 +O 22.90100 4.24000 23.82100 +H 23.20100 3.52000 23.20100 +H 22.51100 3.84000 24.65100 +O 26.74100 15.86100 25.49100 +H 27.06100 15.96100 24.55100 +H 25.95100 15.24100 25.51100 +O 26.32100 13.30100 21.63100 +H 25.86100 12.43100 21.80100 +H 27.23100 13.27100 22.04100 +O 19.01100 10.77100 21.62100 +H 20.00100 10.66100 21.53100 +H 18.61100 9.91100 21.94100 +O 27.37100 16.46100 21.99100 +H 26.60100 16.11100 21.45100 +H 27.05100 17.17100 22.61100 +O 20.92100 24.90100 1.13000 +H 19.99100 24.88100 1.50000 +H 20.93100 24.51100 0.21000 +O 20.87100 21.37100 9.96100 +H 21.22100 21.20100 10.88100 +H 19.99100 20.92100 9.84100 +O 18.81100 22.30100 6.47000 +H 17.99100 22.73100 6.86000 +H 18.53100 21.57100 5.84000 +O 27.12100 26.60100 18.23100 +H 27.08100 27.36100 18.88100 +H 27.34100 26.96100 17.32100 +O 25.47100 0.91100 6.65000 +H 26.16100 0.75100 7.35000 +H 24.74100 1.48100 7.03000 +O 23.00100 22.54100 14.99100 +H 23.82100 21.98100 15.08100 +H 22.19100 21.96100 15.03100 +O 19.84100 25.05100 5.63000 +H 19.39100 24.17100 5.80000 +H 19.83100 25.59100 6.47000 +O 19.75100 25.99100 15.97100 +H 20.63100 25.86100 16.42100 +H 19.62100 26.96100 15.75100 +O 26.71100 18.66100 5.02000 +H 27.11100 19.57100 4.93000 +H 25.71100 18.74100 5.08000 +O 21.79100 21.13100 18.01100 +H 22.50100 21.84100 18.07100 +H 20.91100 21.52100 18.29100 +O 24.60100 25.91100 2.70000 +H 24.84100 26.60100 2.02000 +H 23.82100 26.24100 3.24000 +O 26.69100 24.67100 14.65100 +H 26.22100 24.64100 15.54100 +H 26.18100 24.12100 13.99100 +O 1.20100 25.63100 4.29000 +H 1.46100 25.59100 5.25000 +H 0.35100 25.12100 4.15000 +O 19.38100 26.73100 7.89000 +H 20.37100 26.61100 7.98000 +H 19.14100 27.68100 8.10000 +O 24.12100 20.58100 8.85000 +H 24.07100 20.53100 9.85000 +H 24.14100 21.54100 8.56000 +O 20.64100 21.47100 14.98100 +H 19.84100 22.07100 14.85100 +H 20.54100 20.98100 15.84100 +O 22.44100 25.62100 4.80000 +H 22.89100 24.72100 4.77000 +H 21.50100 25.51100 5.13000 +O 26.43100 21.26100 17.49100 +H 27.10100 20.65100 17.92100 +H 25.70100 21.45100 18.14100 +O 0.01100 23.65100 8.99000 +H -0.23900 23.56100 8.03000 +H 0.49100 22.83100 9.30000 +O 24.01100 19.26100 5.12000 +H 23.20100 19.27100 5.70000 +H 24.04100 20.09100 4.57000 +O 21.59100 18.97100 1.71000 +H 22.08100 19.81100 1.50000 +H 22.21100 18.32100 2.16000 +O 0.14100 20.98100 4.80000 +H -0.33900 21.39100 4.02000 +H 1.13100 20.96100 4.61000 +O 23.21100 2.31100 7.41000 +H 22.50100 2.04100 8.06000 +H 22.95100 2.03100 6.48000 +O 19.40100 24.18100 13.86100 +H 20.32100 24.17100 13.45100 +H 19.34100 24.92100 14.53100 +O 24.23100 20.84100 11.47100 +H 24.61100 20.00100 11.84100 +H 23.35100 21.03100 11.91100 +O 27.28100 23.16100 6.42000 +H 26.96100 23.88100 5.80000 +H 27.52100 22.35100 5.89000 +O 0.96100 19.01100 7.53000 +H 0.24100 19.06100 6.84000 +H 0.72100 18.32100 8.22000 +O 23.89100 21.18100 3.28000 +H 24.16100 20.59100 2.53000 +H 23.89100 22.13100 2.97000 +O 27.77100 19.51100 14.02100 +H 28.02100 19.31100 13.07100 +H 28.49100 20.07100 14.44100 +O 22.72100 26.75100 12.51100 +H 23.58100 26.87100 13.01100 +H 22.30100 25.88100 12.78100 +O 24.28100 23.99100 8.65000 +H 24.40100 24.65100 7.91000 +H 24.74100 24.33100 9.48000 +O 22.64100 26.13100 15.98100 +H 23.32100 26.68100 15.51100 +H 23.04100 25.25100 16.25100 +O 0.92100 19.67100 17.70100 +H 0.98100 20.52100 17.18100 +H 1.24100 18.91100 17.13100 +O 23.66100 19.12100 17.40100 +H 23.24100 18.55100 16.70100 +H 23.00100 19.81100 17.72100 +O 24.35100 27.32100 10.29100 +H 24.79100 28.21100 10.20100 +H 23.72100 27.32100 11.06100 +O 21.69100 20.75100 12.31100 +H 21.46100 21.12100 13.21100 +H 21.39100 19.80100 12.25100 +O 25.94100 24.96100 10.64100 +H 26.53100 24.70100 9.88100 +H 25.66100 25.92100 10.53100 +O 21.69100 19.25100 6.18000 +H 21.58100 20.19100 6.51000 +H 21.64100 18.62100 6.95000 +O 1.02100 26.28100 9.66100 +H 1.17100 26.49100 10.62100 +H 0.72100 25.33100 9.57100 +O 21.81100 26.72100 9.49100 +H 22.74100 27.08100 9.54100 +H 21.75100 25.87100 10.01100 +O 22.01100 23.71100 10.06100 +H 21.49100 22.88100 9.89100 +H 22.78100 23.76100 9.42100 +O 23.73100 22.77100 18.08100 +H 23.55100 23.22100 18.96100 +H 24.15100 23.42100 17.45100 +O 0.81100 26.58100 12.38100 +H 1.22100 26.26100 13.24100 +H -0.14900 26.31100 12.35100 +O 19.02100 24.06100 11.14100 +H 19.87100 23.73100 10.73100 +H 19.15100 24.21100 12.12100 +O 20.51100 23.82100 17.22100 +H 21.10100 23.42100 16.52100 +H 19.93100 24.53100 16.81100 +O 26.77100 24.34100 3.25000 +H 26.84100 23.45100 2.79000 +H 25.83100 24.68100 3.17000 +O 25.33100 23.26100 12.69100 +H 24.99100 22.37100 12.39100 +H 25.59100 23.80100 11.89100 +O 23.35100 23.62100 1.91000 +H 23.96100 24.42100 1.95000 +H 22.40100 23.93100 1.98000 +O 27.65100 19.48100 1.33000 +H 28.16100 19.49100 0.47000 +H 28.21100 19.06100 2.04000 +O 21.25100 21.88100 7.20000 +H 20.46100 22.39100 6.86000 +H 21.16100 21.73100 8.18000 +O 1.18100 1.77100 6.96000 +H 0.48100 1.30100 7.50000 +H 1.77100 1.09100 6.53000 +O 0.93100 20.98100 9.71100 +H 0.85100 20.62100 8.78100 +H 0.91100 20.22100 10.36100 +O 19.67100 0.92100 17.26100 +H 18.90100 0.59100 17.80100 +H 20.52100 0.62100 17.68100 +O 25.34100 20.65100 14.89100 +H 26.24100 20.49100 14.49100 +H 25.42100 20.70100 15.88100 +O 19.34100 20.28100 3.18000 +H 19.17100 21.11100 2.64000 +H 20.24100 19.91100 2.96000 +O 24.75100 27.04100 14.31100 +H 25.31100 27.85100 14.14100 +H 25.34100 26.24100 14.34100 +O 25.78100 24.27100 17.08100 +H 25.97100 24.92100 17.82100 +H 26.38100 23.47100 17.17100 +O 24.56100 26.07100 6.52000 +H 25.06100 26.92100 6.33000 +H 23.68100 26.09100 6.04000 +O 27.21100 0.35100 8.61000 +H 27.75100 -0.33900 9.09000 +H 26.89100 1.04100 9.27000 +O 22.90100 22.86100 5.20000 +H 23.20100 22.14100 4.58000 +H 22.51100 22.46100 6.03000 +O 21.79100 24.09100 12.80100 +H 22.17100 23.50100 13.52100 +H 22.19100 23.83100 11.92100 +O 0.80100 18.96100 11.54100 +H 0.17100 18.24100 11.23100 +H 1.73100 18.60100 11.54100 +O 20.92100 24.90100 19.75100 +H 19.99100 24.88100 20.12100 +H 20.93100 24.51100 18.83100 +O 20.87100 21.37100 0.75100 +H 21.22100 21.20100 1.67100 +H 19.99100 20.92100 0.63100 +O 18.81100 22.30100 25.09100 +H 17.99100 22.73100 25.48100 +H 18.53100 21.57100 24.46100 +O 25.47100 0.91100 25.27100 +H 26.16100 0.75100 25.97100 +H 24.74100 1.48100 25.65100 +O 19.84100 25.05100 24.25100 +H 19.39100 24.17100 24.42100 +H 19.83100 25.59100 25.09100 +O 26.71100 18.66100 23.64100 +H 27.11100 19.57100 23.55100 +H 25.71100 18.74100 23.70100 +O 24.60100 25.91100 21.32100 +H 24.84100 26.60100 20.64100 +H 23.82100 26.24100 21.86100 +O 1.20100 25.63100 22.91100 +H 1.46100 25.59100 23.87100 +H 0.35100 25.12100 22.77100 +O 19.38100 26.73100 26.51100 +H 20.37100 26.61100 26.60100 +H 19.14100 27.68100 26.72100 +O 24.12100 20.58100 27.47100 +H 24.07100 20.53100 28.47100 +H 24.14100 21.54100 27.18100 +O 22.44100 25.62100 23.42100 +H 22.89100 24.72100 23.39100 +H 21.50100 25.51100 23.75100 +O 24.76100 19.84100 19.79100 +H 25.74100 19.62100 19.86100 +H 24.45100 19.67100 18.86100 +O 0.01100 23.65100 27.61100 +H -0.23900 23.56100 26.65100 +H 0.49100 22.83100 27.92100 +O 24.01100 19.26100 23.74100 +H 23.20100 19.27100 24.32100 +H 24.04100 20.09100 23.19100 +O 21.59100 18.97100 20.33100 +H 22.08100 19.81100 20.12100 +H 22.21100 18.32100 20.78100 +O 0.14100 20.98100 23.42100 +H -0.33900 21.39100 22.64100 +H 1.13100 20.96100 23.23100 +O 23.21100 2.31100 26.03100 +H 22.50100 2.04100 26.68100 +H 22.95100 2.03100 25.10100 +O 27.28100 23.16100 25.04100 +H 26.96100 23.88100 24.42100 +H 27.52100 22.35100 24.51100 +O 0.96100 19.01100 26.15100 +H 0.24100 19.06100 25.46100 +H 0.72100 18.32100 26.84100 +O 23.89100 21.18100 21.90100 +H 24.16100 20.59100 21.15100 +H 23.89100 22.13100 21.59100 +O 24.28100 23.99100 27.27100 +H 24.40100 24.65100 26.53100 +H 24.74100 24.33100 28.10100 +O 21.69100 19.25100 24.80100 +H 21.58100 20.19100 25.13100 +H 21.64100 18.62100 25.57100 +O 1.02100 26.28100 0.45100 +H 1.17100 26.49100 1.41100 +H 0.72100 25.33100 0.36100 +O 26.77100 24.34100 21.87100 +H 26.84100 23.45100 21.41100 +H 25.83100 24.68100 21.79100 +O 23.35100 23.62100 20.53100 +H 23.96100 24.42100 20.57100 +H 22.40100 23.93100 20.60100 +O 21.25100 21.88100 25.82100 +H 20.46100 22.39100 25.48100 +H 21.16100 21.73100 26.80100 +O 27.78100 27.72100 21.53100 +H 28.41100 28.10100 20.85100 +H 28.18100 26.89100 21.92100 +O 1.18100 1.77100 25.58100 +H 0.48100 1.30100 26.12100 +H 1.77100 1.09100 25.15100 +O 19.37100 22.07100 18.95100 +H 18.45100 21.79100 18.66100 +H 19.68100 22.84100 18.39100 +O 19.34100 20.28100 21.80100 +H 19.17100 21.11100 21.26100 +H 20.24100 19.91100 21.58100 +O 24.56100 26.07100 25.14100 +H 25.06100 26.92100 24.95100 +H 23.68100 26.09100 24.66100 +O 1.62100 0.63100 19.49100 +H 1.39100 1.16100 18.67100 +H 2.43100 0.07100 19.30100 +O 22.90100 22.86100 23.82100 +H 23.20100 22.14100 23.20100 +H 22.51100 22.46100 24.65100 diff --git a/examples/gpmdk/run/coords_300.dat b/examples/gpmdk/run/coords_300.dat new file mode 100644 index 00000000..6eec1584 --- /dev/null +++ b/examples/gpmdk/run/coords_300.dat @@ -0,0 +1,304 @@ + 300 + 14.40500 0.00000 0.00000 + 0.00000 14.40500 0.00000 + 0.00000 0.00000 14.40500 +H 2.44100 0.20600 1.00300 +O 2.28300 0.06800 0.04900 +H 1.79100 13.63200 0.01400 +H 0.50700 14.25000 6.56400 +O 0.89500 0.74100 6.43800 +H 0.12600 1.30400 6.65900 +H 6.07100 4.56800 13.85400 +O 6.37600 5.36600 14.36300 +H 6.85400 4.94600 0.72800 +H 9.13800 4.82900 5.13900 +O 9.29500 4.91700 4.17100 +H 10.26100 4.85100 4.00400 +H 8.42200 9.67800 11.47900 +O 7.63900 10.22400 11.24900 +H 6.98500 10.03000 11.95900 +H 6.55500 7.29100 3.44600 +O 7.45600 7.68200 3.53200 +H 7.49900 7.84300 4.50800 +H 4.03200 0.78800 4.74000 +O 4.75200 0.59500 5.39200 +H 4.93100 14.05400 5.27900 +H 1.04800 12.26800 2.34300 +O 1.88300 12.29500 1.83400 +H 1.86500 11.48000 1.30400 +H 2.74800 5.73400 0.76500 +O 3.46700 5.76700 0.11100 +H 3.92400 4.91000 0.17200 +H 14.40100 8.01500 4.04100 +O 0.76200 8.54100 3.68700 +H 1.00500 8.03800 2.88300 +H 7.57100 5.54500 7.83500 +O 7.65800 5.22700 6.90000 +H 7.10000 4.43000 6.98000 +H 12.14000 7.26200 7.79000 +O 13.06300 7.56900 7.96500 +H 13.60300 6.79600 7.66600 +H 5.93400 8.72200 10.97800 +O 5.13500 8.66700 11.53300 +H 4.54900 8.03000 11.07400 +H 7.15000 0.49500 4.78800 +O 8.03300 0.64100 5.18500 +H 8.68400 0.54600 4.46200 +H 1.35800 11.09600 7.49700 +O 1.66800 11.00200 6.56900 +H 1.84200 10.05600 6.42900 +H 2.32300 0.33300 7.72900 +O 3.22700 14.38400 7.87200 +H 3.04600 13.58400 8.41900 +H 9.01000 4.35900 10.22700 +O 9.93700 4.62600 10.06700 +H 10.04200 4.45800 9.11500 +H 2.06900 2.69100 13.04500 +O 2.59600 2.79900 13.86800 +H 2.55700 1.89200 14.23700 +H 5.72700 4.79800 4.39600 +O 6.51700 5.06700 3.85800 +H 7.25200 4.96500 4.49800 +H 2.69800 5.51700 7.46500 +O 3.02200 6.39100 7.77500 +H 3.46900 6.21700 8.63900 +H 0.76700 4.55300 4.59100 +O 0.38900 3.83000 4.05200 +H 14.34500 4.32900 3.33800 +H 12.18900 4.33400 10.12700 +O 12.65400 4.89700 10.77400 +H 12.54400 4.41900 11.61900 +H 8.10500 13.66900 2.56500 +O 8.43200 13.12200 3.29900 +H 7.81600 12.42300 3.59800 +H 13.16500 9.90100 12.67200 +O 14.15900 9.90100 12.39300 +H 14.27000 9.05600 11.92800 +H 7.84000 13.93900 9.33300 +O 7.92300 14.23400 10.25800 +H 7.40000 13.58600 10.76200 +H 6.28400 0.41000 8.01800 +O 5.69000 1.17500 8.11800 +H 4.84100 0.82000 7.71700 +H 1.94900 12.85000 10.69600 +O 2.01500 12.54100 9.76500 +H 1.28400 11.89400 9.72200 +H 14.31000 7.21700 10.10700 +O 0.47300 7.78600 10.64900 +H 1.34500 7.36400 10.58100 +H 1.91600 0.79200 3.31800 +O 2.78500 0.62900 2.89000 +H 3.06600 14.17500 3.29700 +H 10.53100 2.50700 2.36700 +O 11.36600 2.91800 2.69700 +H 11.56000 2.47700 3.54000 +H 3.27600 5.45800 11.53400 +O 4.15400 5.85000 11.33800 +H 4.43800 6.12500 12.23200 +H 7.00100 8.93100 9.33700 +O 7.39500 8.40400 8.62600 +H 7.02100 7.49300 8.72600 +H 5.75200 5.35800 9.37400 +O 6.56800 5.90700 9.32100 +H 6.88200 5.97000 10.27200 +H 10.40400 12.64500 6.10700 +O 10.85000 13.13700 5.38900 +H 10.99800 12.46100 4.68700 +H 10.61900 10.58000 13.77000 +O 11.59400 10.52100 13.96100 +H 11.70300 9.55700 14.13200 +H 4.10700 1.50300 1.76500 +O 4.94200 1.40000 1.27600 +H 4.85000 0.48400 0.94400 +H 7.95300 12.37300 7.74500 +O 8.51700 12.17400 8.51900 +H 8.04200 11.47200 8.99800 +H 8.65200 3.68700 0.28100 +O 9.36200 3.39000 14.09900 +H 8.93600 2.79700 13.48000 +H 0.50300 8.64900 13.75200 +O 1.28100 8.48500 14.34500 +H 1.02900 7.68600 0.48100 +H 4.43000 12.76400 9.75100 +O 4.82500 13.52700 10.22600 +H 5.21400 14.07100 9.51700 +H 2.58000 9.53500 9.06300 +O 3.17400 8.77600 8.89400 +H 3.46100 8.85100 7.96600 +H 4.11900 12.68100 13.25000 +O 4.22500 12.06400 12.48900 +H 3.84800 11.22300 12.85000 +H 3.71200 13.00100 0.99200 +O 4.62800 12.95800 0.64700 +H 4.98900 12.13800 1.10600 +H 7.56200 9.50400 3.04100 +O 8.04500 10.34400 3.25300 +H 9.00100 10.12000 3.12500 +H 6.65100 11.71200 2.12800 +O 5.77600 11.41900 2.45900 +H 5.84800 10.44500 2.46400 +H 12.43900 6.66900 1.83900 +O 11.90900 6.01200 2.32300 +H 12.22900 5.14600 2.00300 +H 0.01500 5.86600 1.71600 +O 0.86300 6.29800 1.95400 +H 1.47200 5.58300 2.19800 +H 11.93100 12.50200 8.47200 +O 11.09900 12.76900 8.03300 +H 10.40400 12.33000 8.55900 +H 4.18500 7.37800 6.80400 +O 4.58800 8.16400 6.36200 +H 4.06900 8.26300 5.54600 +H 13.90600 2.01700 8.61300 +O 0.42000 2.40200 8.55700 +H 0.72200 2.32600 9.49200 +H 4.77800 3.30700 11.98200 +O 5.22100 2.98700 12.79900 +H 4.50400 3.06700 13.48000 +H 13.34400 13.91800 3.35700 +O 14.12600 14.28700 3.81400 +H 0.32500 13.51800 3.91800 +H 9.03700 6.80200 13.69500 +O 9.60000 7.46700 13.23300 +H 9.42900 7.30100 12.27400 +H 0.50800 11.30700 5.18600 +O 0.38100 11.38200 4.20800 +H 0.87200 10.59300 3.84800 +H 12.97800 3.42400 1.81800 +O 13.70400 3.93900 1.38500 +H 13.84500 3.49900 0.52700 +H 12.19600 6.94000 4.04700 +O 13.00400 7.21100 4.53000 +H 12.67700 7.39900 5.42900 +H 10.42200 0.27300 11.89100 +O 10.91400 0.38200 11.04600 +H 10.21900 0.31100 10.36600 +H 7.67500 2.21900 8.04800 +O 8.31300 2.58900 7.40300 +H 8.25800 1.94000 6.64500 +H 10.07400 10.19500 6.81700 +O 9.89800 10.16400 5.86100 +H 10.36600 9.35900 5.57300 +H 12.13400 1.07200 9.93600 +O 12.23300 1.27900 8.95600 +H 12.27200 0.42600 8.41600 +H 4.05900 7.71300 1.79200 +O 4.71500 7.10600 2.16000 +H 4.64800 6.38100 1.50800 +H 13.87700 10.61100 8.87100 +O 0.38900 10.36300 9.16200 +H 0.24000 9.46200 9.49100 +H 2.96700 9.01700 14.01200 +O 3.78500 9.58100 13.91400 +H 4.27700 9.04900 13.24700 +H 10.08500 14.00000 2.85900 +O 10.87300 14.37900 2.55900 +H 11.33800 0.39000 3.32600 +H 11.30900 7.53200 14.04400 +O 12.22900 7.70000 14.38900 +H 12.79700 7.04500 13.91200 +H 6.46100 13.93100 0.93500 +O 7.39600 13.67800 0.82500 +H 7.40600 13.36300 14.29800 +H 12.03000 1.44800 12.36400 +O 12.79400 1.37100 12.95000 +H 12.48500 0.71800 13.59900 +H 2.12200 3.25800 7.72200 +O 2.41000 3.77400 6.94200 +H 1.76400 3.47600 6.27600 +H 4.22800 3.98700 6.00400 +O 5.09600 3.72200 5.62800 +H 4.96000 2.80100 5.32800 +H 6.04700 9.52000 6.08700 +O 6.21900 10.28600 5.51600 +H 7.18900 10.39800 5.52700 +H 8.77800 12.11800 0.33800 +O 8.80000 11.14300 0.34500 +H 8.72500 10.88600 1.28400 +H 10.10700 3.02100 7.58500 +O 10.98000 3.50500 7.61600 +H 11.55900 2.82600 8.08000 +H 3.81500 0.37900 11.05900 +O 3.29900 1.19700 10.81100 +H 3.13700 1.04300 9.85600 +H 13.45000 5.79100 12.68600 +O 14.08500 6.44400 13.02200 +H 0.43100 6.35200 12.39500 +H 3.42700 12.06000 4.47100 +O 3.08300 12.95500 4.64600 +H 2.46000 12.84000 5.38600 +H 13.38900 10.00500 5.93700 +O 13.06200 10.09100 6.85900 +H 13.13100 9.17900 7.24100 +H 9.11400 7.98200 9.50300 +O 9.51600 7.58100 10.31200 +H 9.61800 6.62100 10.11700 +H 4.86000 3.20900 9.65100 +O 4.91300 3.71100 8.81700 +H 5.20200 3.03100 8.17800 +H 11.40400 9.36900 2.31000 +O 10.88500 9.86400 2.97000 +H 11.37800 9.78200 3.81300 +H 7.23000 3.78400 2.58200 +O 7.22100 3.20900 1.77800 +H 6.52200 2.53200 1.91800 +H 9.77400 8.20900 3.61000 +O 10.15200 7.57100 4.25100 +H 9.45400 6.89400 4.34800 +H 12.73400 13.13300 6.28600 +O 13.58500 13.10700 6.77900 +H 13.71100 12.13200 6.92500 +H 10.94300 4.25600 13.77000 +O 11.64400 4.91500 13.57600 +H 11.18700 5.75300 13.76000 +H 13.17500 4.66400 6.75600 +O 13.93700 4.92600 7.30800 +H 0.00800 4.08900 7.49900 +H 9.23200 0.02700 14.11100 +O 10.09800 14.15000 13.77100 +H 10.64300 14.14700 0.17500 +H 11.54100 0.53900 5.36700 +O 12.05300 1.30000 4.99700 +H 12.97700 0.98000 4.91300 +H 10.54600 11.50500 11.84100 +O 11.22400 11.83900 11.21900 +H 11.17800 12.81700 11.31600 +H 9.91800 6.03300 6.35500 +O 10.79300 6.15800 6.76000 +H 10.87500 5.36000 7.33500 +H 6.86300 2.48500 10.77100 +O 7.80800 2.52200 11.06600 +H 8.10000 1.59000 10.91800 +H 5.25000 12.79800 6.97500 +O 6.22400 12.78300 6.96300 +H 6.45400 12.10200 6.30800 +H 1.87300 3.69600 2.82600 +O 2.60200 3.96600 2.23300 +H 3.06200 3.14900 1.99600 +H 12.48600 12.31900 0.01700 +O 12.77300 13.11700 0.52100 +H 11.96200 13.45700 0.95200 +H 6.19400 12.38600 12.57000 +O 7.09800 12.71300 12.49900 +H 7.62200 12.00600 12.02800 +H 0.51200 1.65100 12.21400 +O 0.95800 2.07500 11.45200 +H 1.85400 1.62600 11.33700 +H 0.03200 13.04100 12.91800 +O 0.43200 12.61800 12.10900 +H 0.18700 11.64900 12.11500 +H 9.77600 5.76100 1.70900 +O 8.80000 5.83300 1.60300 +H 8.52300 6.45400 2.32500 +H 7.31400 5.72700 12.84400 +O 7.63500 5.59900 11.90600 +H 7.84700 4.59300 11.87700 +H 5.53800 9.45400 0.42400 +O 6.40700 9.29200 0.86200 +H 6.27900 8.38500 1.19900 +H 4.40000 10.34500 9.33100 +O 4.49200 11.09000 8.70400 +H 4.12200 10.74300 7.87400 +H 0.08700 4.80700 10.70500 +O 0.94000 4.68900 11.19200 +H 0.90200 3.73400 11.48000 diff --git a/examples/gpmdk/run/get_energy.py b/examples/gpmdk/run/get_energy.py new file mode 100755 index 00000000..2da423a3 --- /dev/null +++ b/examples/gpmdk/run/get_energy.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import argparse +import re + +energy_re = re.compile("Energy Total.*=\s+([0-9-.]+)") + +parser = argparse.ArgumentParser() +parser.add_argument("OUT", + help="The output") +options = parser.parse_args() + +with open(options.OUT) as fd: + for line in fd: + result = energy_re.search(line) + if result: + print(result.group(1)) diff --git a/examples/gpmdk/run/gpmd_input.in b/examples/gpmdk/run/gpmd_input.in new file mode 100644 index 00000000..2ff3af1d --- /dev/null +++ b/examples/gpmdk/run/gpmd_input.in @@ -0,0 +1,153 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 10 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + CoordsFile= "CH4.pdb" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.1 + #TimeStep= 0.00 + MDSteps= 200000 + ParamPath= "../sulfurTBparam" + NlistEach= 1 + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.025 + Entropy= T + DoKernel= F +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + #GraphElement= Orbital + #PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + #PartitionType= METIS + PartitionType= Block + NodesPerPart= 0 + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 4 + PartitionCount= 1 + #PartitionCount= 512 + #PartitionCount= 16 + #PartitionCount= 2 + #PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 32 + #PartitionCount= 64 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= -1 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + BuildAlways= F + RankNUpdate= 3 + KernelMixing= F + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 1 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + WriteCoordsEach= 1 + #TrajectoryFormat= XYZ + InitialTemperature= 300 + #HtoD= T + MinimizationSteps= 100 + LangevinDynamics= T + LangevinMethod= Siva + UseSMD= F + #For polyaniline + #SMDAtomIndex1= 6 + #SMDAtomIndex2= 7 + + SMDForceConstant= -8.0 + SMDR0= 0.001 + ComputeCurrents= T + CurrentThreshold= 0.0 +} diff --git a/examples/gpmdk/run/input.in b/examples/gpmdk/run/input.in new file mode 100644 index 00000000..e85741ee --- /dev/null +++ b/examples/gpmdk/run/input.in @@ -0,0 +1,182 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + #StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.2 + #TimeStep= 0.00 + MDSteps= 10000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + #CoordsFile= coords_300.dat + CoordsFile= coords_2088.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb + NlistEach= 1 + #BMLDistributionType= Distributed + MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn + EFermi= -0.0 + #kBT= 0.04308695 + kBT= 0.025 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + #PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + PartitionType= METIS + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 256 + #PartitionCount= 512 + #PartitionCount= 16 + #PartitionCount= 1 + PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 32 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= -1 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks + BuildAlways= F + RankNUpdate= 4 + KernelMixing= T + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 1 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + WriteCoordsEach= 10 + LangevinMethod= Siva + LangevinDynamics= T + LangevinGamma= 0.01 + InitialTemperature= 300.0 +} diff --git a/examples/gpmdk/run/job b/examples/gpmdk/run/job new file mode 100755 index 00000000..1ecc0cf0 --- /dev/null +++ b/examples/gpmdk/run/job @@ -0,0 +1,19 @@ +#!/bin/bash + +#SBATCH -N32 #Number of cores and nodes +##SBATCH -N1 #Number of cores and nodes +#SBATCH --ntasks-per-node=72 +#SBATCH -p scaling --time=8:00:00 +# OMP_NUM_THREADS=36 controls the number of threads your application use +## This variable cat be set by the following command : + +source $HOME/BESGraph/qmd-progress/examples/gpmdcov/vars + +## Run command +export OMP_NUM_THREADS=36 +RUN="$HOME/BESGraph/qmd-progress/build/gpmdcov" +#OMP_PROC_BIND=spread OMP_NUM_THREADS=4 mpirun -n 128 --map-by ppr:8:node:pe=4 $RUN input.in > out +#OMP_PROC_BIND=spread OMP_NUM_THREADS=36 mpirun -n 1 --map-by ppr:4:node:pe=36 $RUN input.in > out +#OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 64 --map-by ppr:4:node:pe=9 $RUN input.in > out +OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 16 --map-by ppr:4:node:pe=9 $RUN input.in > out + diff --git a/examples/gpmdk/run/latte.in b/examples/gpmdk/run/latte.in new file mode 100644 index 00000000..59947cd5 --- /dev/null +++ b/examples/gpmdk/run/latte.in @@ -0,0 +1,74 @@ +LATTE INPUT FILE +================ +#This input file resumes the content of MDcontroller and TBparam/control.in +#The parser will only read it if it is present inside the running folder. +#In case this file is not present Latte will read the two files as original. +#The order of the kewords is not important in this file. + +#General controls +CONTROL{ + XCONTROL= 1 + BASISTYPE= NONORTHO + PARAMPATH= "./latteTBparams" + DEBUGON= 0 + FERMIM= 6 + CGORLIB= 1 CGTOL= 1.0e-6 + KBT= 0.0 + NORECS= 1 + ENTROPYKIND= 1 + PPOTON= 1 VDWON= 0 + SPINON= 0 SPINTOL= 1.0e-4 + ELECTRO= 0 ELECMETH= 1 ELEC_ETOL= 0.001 ELEC_QTOL= 1.0e-4 + COULACC= 1.0e-6 COULCUT= -500.0 COULR1= 500.0 + MAXSCF= 250 + BREAKTOL= 1.0E-12 MINSP2ITER= 22 SP2CONV= REL + FULLQCONV= 1 QITER= 0 + QMIX= 0.25 SPINMIX= 0.25 MDMIX= 0.25 + ORDERNMOL= 0 + SPARSEON= 0 THRESHOLDON= 1 NUMTHRESH= 1.0e-6 FILLINSTOP= 100 BLKSZ= 4 + MSPARSE= 3000 + LCNON= 0 LCNITER= 4 CHTOL= 0.01 + SKIN= 1.0 + RELAX= 0 RELAXTYPE= SD MAXITER= 100 RLXFTOL= 0.00001 + MDON= 0 + PBCON= 1 + RESTART= 0 + CHARGE= 0 + XBO= 1 + XBODISON= 1 + XBODISORDER= 5 + NGPU= 2 + KON= 0 + COMPFORCE= 1 + DOSFIT= 0 INTS2FIT= 1 BETA= 1000.0 NFITSTEP= 5000 QFIT= 0 MCSIGMA= 0.2 + PPFITON= 0 + ALLFITON= 0 + PPSTEP= 500 BISTEP= 500 PP2FIT= 2 BINT2FIT= 6 + PPBETA= 1000.0 PPSIGMA= 0.01 PPNMOL= 10 PPNGEOM= 200 + PARREP= 0 + ER= 1.0 +} + +#Controls for QMD +MDCONTROL{ + MAXITER= 20 + UDNEIGH= 1 + DT= 0.25 + TEMPERATURE= 1.0e-30 RNDIST= GAUSSIAN SEEDINIT= UNIFORM + DUMPFREQ= 250 + RSFREQ= 500 + WRTFREQ= 1 + TOINITTEMP5= 1 + THERMPER= 500 + THERMRUN= 50000 + NVTON= 0 NPTON= 0 AVEPER= 1000 FRICTION= 1000.0 SEED= 54 + PTARGET= 0.0 NPTTYPE= ISO + SHOCKON= 0 + SHOCKSTART= 100000 + SHOCKDIR= 1 + UPARTICLE= 500.0 USHOCK= -4590.0 C0= 1300.0 + MDADAPT= 0 + GETHUG= 0 E0= -795.725 V0= 896.984864 P0= 0.083149 +} + + diff --git a/examples/gpmdk/run/run.sh b/examples/gpmdk/run/run.sh new file mode 100755 index 00000000..0696145d --- /dev/null +++ b/examples/gpmdk/run/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + +#RUN="$PWD/../build_hackathon/gpmdk" +#OMP_NUM_THREADS=32 mpiexec -np 3 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpirun -np 1 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpiexec -np 1 $RUN input.in | tee out + +#OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out +OMP_NUM_THREADS=32 srun -n 4 -c 32 bash wrapper.sh +#OMP_NUM_THREADS=64 $RUN input.in | tee out +#OMP_NUM_THREADS=36 $RUN input.in | tee out +#$RUN input.in | tee out + diff --git a/examples/gpmdk/run/sbatch.sh b/examples/gpmdk/run/sbatch.sh new file mode 100755 index 00000000..27a1b9e7 --- /dev/null +++ b/examples/gpmdk/run/sbatch.sh @@ -0,0 +1,20 @@ +#!/bin/bash +##SBATCH -p gpu +#SBATCH -p gpu_debug +#SBATCH --reservation=gpu_debug +#SBATCH --qos=debug +##SBATCH --time 16:00:00 +#SBATCH --time 2:00:00 +#SBATCH -N 2 +##SBATCH -A ichelp_g +#SBATCH -A w23_macroqmd_g + +cd ${PWD} + +source /usr/projects/icapt/mewall/packages/gpmd/gpmd/setenv_gpu.sh + +#export MPICH_ALLREDUCE_NO_SMP=1 +export MPICH_SMP_SINGLE_COPY_MODE=NONE + +OMP_NUM_THREADS=32 srun -n 8 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh + diff --git a/examples/gpmdk/run/wrapper.sh b/examples/gpmdk/run/wrapper.sh new file mode 100644 index 00000000..2a0f668e --- /dev/null +++ b/examples/gpmdk/run/wrapper.sh @@ -0,0 +1,10 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) + +RUN="$PWD/../build_hackathon/gpmdk" +#stdbuf -o0 $RUN input_ICH.in |& tee out_ICH +if [ $SLURM_PROCID -eq 0 ]; then + $RUN input.in >& out +else + $RUN input.in >& /dev/null +fi diff --git a/examples/gpmdk/setenv_chicoma_hackathon.sh b/examples/gpmdk/setenv_chicoma_hackathon.sh new file mode 100644 index 00000000..3b1a6a37 --- /dev/null +++ b/examples/gpmdk/setenv_chicoma_hackathon.sh @@ -0,0 +1,6 @@ +module switch PrgEnv-cray PrgEnv-gnu +module swap gcc gcc/11.2.0 +export PKG_CONFIG_PATH=/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig:$PKG_CONFIG_PATH +module load cmake +module load cudatoolkit +module load intel-mkl diff --git a/examples/gpmdk/src/CMakeLists.txt b/examples/gpmdk/src/CMakeLists.txt new file mode 100644 index 00000000..dfaf40e1 --- /dev/null +++ b/examples/gpmdk/src/CMakeLists.txt @@ -0,0 +1,209 @@ +cmake_minimum_required(VERSION 3.10.0) +project(appendix C CXX Fortran) + +set(dir ${CMAKE_CURRENT_SOURCE_DIR}/build/) +set(CMAKE_BUILD_DIRECTORY ${dir}) +set(CMAKE_CURRENT_BINARY_DIR ${dir}) + +include(FindPkgConfig) + +find_package(PROGRESS CONFIG QUIET) +pkg_check_modules(PROGRESS REQUIRED progress) +message(STATUS "Found progress: ${PROGRESS_LDFLAGS}") +list(APPEND LINK_LIBRARIES ${PROGRESS_LDFLAGS}) + +find_package(BML CONFIG QUIET) +pkg_check_modules(BML REQUIRED bml) +list(APPEND LINK_LIBRARIES BML::bml) +list(APPEND LINK_LIBRARIES ${BML_LDFLAGS}) +message(STATUS "Found bml: ${BML_LDFLAGS}") + +find_library(FOUND_METIS metis) + if(NOT FOUND_METIS) + message(FATAL_ERROR "Could not find metis library") + endif() + message(STATUS "Found metis: ${FOUND_METIS}") + add_definitions(-DDO_GRAPHLIB) + list(APPEND LINK_LIBRARIES ${FOUND_METIS}) + + get_filename_component(METIS_LIB ${FOUND_METIS} DIRECTORY) + list(APPEND LINK_LIBRARIES "-L/${METIS_LIB} -lmetis") + +if(PROGRESS_MPI) + message(STATUS "Will build with MPI") + add_definitions(-DDO_MPI) +endif() + +if(SANITY_CHECK) + message(STATUS "Will build with SANITY CHECK") + add_definitions(-DSANITY_CHECK) +endif() + +message(STATUS "Extra FC Flags ${EXTRA_FCFLAGS}") + +if(DEFINED EXTRA_FCFLAGS) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${EXTRA_FCFLAGS}") +endif() + + +if(LIB) + list(APPEND SHAREDLIB "-fPIC -shared") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${EXTRA_FCFLAGS} ${SHAREDLIB}") +endif() + +message(STATUS "Project sources = " ${PROJECT_SOURCE_DIR} ) +include_directories(${CMAKE_BINARY_DIR}/) +include_directories(${BML_INCLUDEDIR}) +include_directories(${PROGRESS_INCLUDEDIR}) + +#add_subdirectory(hamiltonian) +#set(dir "${CMAKE_CURRENT_SOURCE_DIR}/hamiltonian") +if(LATTE) + set(extras) + add_definitions(-DUSE_LATTE) + message(STATUS "Will build with LATTE modules in" "${LATTE_DIR}") + if(${LATTE_PREC} STREQUAL "SINGLE") + add_definitions(-DSINGLEPREC) + else() + add_definitions(-DDOUBLEPREC) + endif() + set(dir "${LATTE_DIR}/src/") + +list( + APPEND extras + "${dir}/setuparray.F90" + "${dir}/myprecision.F90" + "${dir}/constants_mod.F90" + "${dir}/readtb.F90" + "${dir}/spinarray.F90" + "${dir}/restartarray.F90" + "${dir}/kspacearray.F90" + "${dir}/neblistarray.F90" + "${dir}/readcr.F90" + "${dir}/xboarray.F90" + "${dir}/nonoarray.F90" + "${dir}/univarray.F90" + "${dir}/ppotarray.F90" + "${dir}/virialarray.F90" + "${dir}/mdarray.F90" + "${dir}/univscaling.F90" + "${dir}/univscaling_function.F90" + "${dir}/wignerd.F90" + "${dir}/wignerarray.F90" + "${dir}/univtailcoef.F90" + "${dir}/latteparser_latte_mod.F90" + "${dir}/wignerd.F90" + "${dir}/wignerarray.F90" + "${dir}/univtailcoef.F90" + "${dir}/latteparser_latte_mod.F90" + "${dir}/coulombarray.F90" + "${dir}/sparsearray.F90" + "${dir}/relaxcommon.F90" + "${dir}/openfiles_mod.F90" + "${dir}/kernelparser_mod.F90" + "${dir}/fermicommon.F90" + "${dir}/errors.F90" + "${dir}/timer_mod.F90" + "${dir}/gradH.F90" + "${dir}/dfdx.F90" + "${dir}/am.F90" + "${dir}/bm.F90" + "${dir}/dunivscaling.F90" + "${dir}/gethdim.F90" + "${dir}/gencutofflist.F90" + "${dir}/build_integral_map.F90" + "${dir}/bldnewH.F90" + "${dir}/genX.F90" + "${dir}/angfactor.F90" + "${dir}/genXprogress.F90" + "${dir}/allocatenono.F90" + "${dir}/genorbitallist.F90" + "${dir}/getmatindlist.F90" + "${dir}/neblists.F90" + "${dir}/pbc.F90" + "${dir}/genHonsite.F90" + "${dir}/allocatenebarrays.F90" + "${dir}/deallocatenono.F90" + "${dir}/deallocatenebarrays.F90" +) +endif() + + +list( + APPEND common_sources +"gpmdcov_allocation.F90" +"gpmdcov_buildz.F90" +"gpmdcov_constraints.F90" +"gpmdcov_diagonalize.F90" +"gpmdcov_dm_min.F90" +"gpmdcov_dump.F90" +"gpmdcov_energandforces.F90" +"gpmdcov_finalize.F90" +"gpmdcov_firstcharges.F90" +"gpmdcov_graphpart.F90" +"gpmdcov_init.F90" +"gpmdcov_initparts.F90" +"gpmdcov_kernel.F90" +"gpmdcov_langevin.F90" +"gpmdcov_mdloop.F90" +"gpmdcov_assert.F90" +"gpmdcov_mod.F90" +"gpmdcov_part.F90" +"gpmdcov_preparemd.F90" +"gpmdcov_reshuffle.F90" +"gpmdcov_restart.F90" +"gpmdcov_rhosolver.F90" +"gpmdcov_test.F90" +"gpmdcov_vars.F90" +"gpmdcov_writeout.F90" +"gpmdcov_neighbor.F90" +"gpmdcov_parser.F90" +"gpmdcov_latte.F90" +"gpmdcov_rhosolver.F90" +"gpmdcov_nonequilibrium.F90" +"gpmdcov_constraints.F90" +"gpmdcov_highorder.F90" +"gpmdcov_field.F90" + ) + + +function(progress_appendix myappendix main_and_srcs) +list(GET main_and_srcs 0 main) +include_directories(${PROGRESS_INCLUDEDIR}) +add_executable(${myappendix} ${main} ${common_sources} ${extras}) +target_sources(${myappendix} PRIVATE ${ARGN}) +target_link_libraries(${myappendix} PUBLIC + ${LINK_LIBRARIES}) + set_target_properties(${myappendix} + PROPERTIES + LINK_FLAGS "") + #add_subdirectory(hamiltonian) +endfunction(progress_appendix) + +function(progress_appendix_library myappendix main_and_srcs) +list(GET main_and_srcs 0 main) +include_directories(${PROGRESS_INCLUDEDIR}) +add_library(${myappendix} SHARED ${main} ${common_sources} ${extras}) +target_sources(${myappendix} PRIVATE ${ARGN}) +target_link_libraries(${myappendix} PUBLIC + ${LINK_LIBRARIES}) + set_target_properties(${myappendix} + PROPERTIES + LINK_FLAGS "") +endfunction(progress_appendix_library) + +if(LIB) + progress_appendix_library( gpmd gpmdcov_lib.F90 gpmdcov_lib_mod.F90 ) + #progress_appendix( gpmd_a gpmd_secuential.F90 gpmdcov_lib_mod.F90) + file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/gpmd.py + DESTINATION ${PROJECT_BINARY_DIR}) +else() + progress_appendix( gpmdk gpmdk.F90 ) + progress_appendix( gpmd_a gpmd_secuential.F90 gpmdcov_lib_mod.F90) +endif() + +SET(DESTINATION ${PROJECT_BINARY_DIR}) +install(FILES ${CMAKE_BINARY_DIR}/progress.pc + DESTINATION ${PROJECT_BINARY_DIR}) + +SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}) diff --git a/examples/gpmdk/src/gpmd.py b/examples/gpmdk/src/gpmd.py new file mode 100755 index 00000000..e7c05e1f --- /dev/null +++ b/examples/gpmdk/src/gpmd.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import numpy as np +from sdc_system import * +from sdc_ptable import ptable +import ctypes as ct +import os + +## General gpmd call function +# @brief This will run gpmd and get back forces and charges, ect. +# from a full SCF optimization. Other output could be added. +# @param LatticeVectors. 3x3 matrix containing the lattice vectors for the simulation box. +# latticeVectors[1,:] = first lattice vector. +# @param symbols Symbols for each atom type, e.g., the element symbol of the first atom is symbols[types[0]] +# @param atomTypes Type for each atom, e.g., the first atom is of type "types[0]" +# @param coords for each atom, e.g., z-coordinate of the frist atom is coords[0,2] +# @param verb Verbosity level for gpmd output +# +def gpmd(latticeVectors,symbols,atomTypes,coords,verb): + + # Import the shared library + gpmdLibFileName = os.environ['GPMD_PATH'] + '/libgpmd.so' + + try: + gpmdLib = ct.CDLL(gpmdLibFileName) + f = gpmdLib.gpmd_compute + except: + print("ERROR loading the gpmd library. Please the check that the \ + GPMD_PATH is set correctly and that there is a compiled libgpmd.so") + exit(0) + + #Periodic table: We use this to pass the chemical atom types as integer instead of characters. + pt = ptable() + + nats = len(coords[:,1]) + + #Getting atomic numbers + nTypes = len(symbols) + atomicNumbers = np.zeros((nTypes),dtype=np.int32) + atomTypes32 = np.zeros((nats),dtype=np.int32) + atomTypes32[:] = atomTypes + for i in range(len(symbols)): + atomicNumbers[i] = pt.get_atomic_number(symbols[i]) + + + #Vectorizing 2D arrays for C-Fortran interoperability + coordsFlat_in = np.zeros(3*nats) #Vectorized coordinates + forcesFlat_out = np.zeros(3*nats) #Vectorized forces + chargesFlat_out = np.zeros(nats) #We call this one Flat just for consistency + dipoleFlat_out = np.zeros(3) #Same here + bornchFlat_out = np.zeros(9*nats) + + for i in range(nats): + coordsFlat_in[3*i] = coords[i,0] + coordsFlat_in[3*i+1] = coords[i,1] + coordsFlat_in[3*i+2] = coords[i,2] + + latticeVectorsFlat = np.zeros((9)) + latticeVectorsFlat[0] = latticeVectors[0,0] + latticeVectorsFlat[1] = latticeVectors[0,1] + latticeVectorsFlat[2] = latticeVectors[0,2] + + latticeVectorsFlat[3] = latticeVectors[1,0] + latticeVectorsFlat[4] = latticeVectors[1,1] + latticeVectorsFlat[5] = latticeVectors[1,2] + + latticeVectorsFlat[6] = latticeVectors[2,0] + latticeVectorsFlat[7] = latticeVectors[2,1] + latticeVectorsFlat[8] = latticeVectors[2,2] + + #Specify arguments as a pointers to pass to Fortran + f.argtypes=[ct.c_int,ct.c_int,ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),\ + ct.POINTER(ct.c_int),ct.POINTER(ct.c_int),ct.POINTER(ct.c_double),\ + ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),ct.c_int] + + #Inputs + coords_ptr = coordsFlat_in.ctypes.data_as(ct.POINTER(ct.c_double)) + atomTypes_ptr = atomTypes32.ctypes.data_as(ct.POINTER(ct.c_int)) + atomicNumbers_ptr = atomicNumbers.ctypes.data_as(ct.POINTER(ct.c_int)) + latticeVectors_ptr = latticeVectorsFlat.ctypes.data_as(ct.POINTER(ct.c_double)) + + #Outputs + charges_ptr = chargesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + forces_ptr = forcesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + dipole_ptr = dipoleFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + bornch_ptr = bornchFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + + #Call to the fortran funtion + err = f(ct.c_int(nats),ct.c_int(nTypes),coords_ptr,latticeVectors_ptr,\ + atomTypes_ptr,atomicNumbers_ptr,charges_ptr,forces_ptr,\ + dipole_ptr,bornch_ptr,ct.c_int(verb)) + + charges_out = np.zeros(nats) + charges_out[:] = chargesFlat_out[:] + dipole_out = np.zeros(3) + dipole_out[:] = dipoleFlat_out[:] + + #Back to a 2D array for the forces + forces_out = np.zeros((nats,3)) + bornch_out = np.zeros((nats,9)) + for i in range(nats): + forces_out[i,0] = forcesFlat_out[i*3 + 0] + forces_out[i,1] = forcesFlat_out[i*3 + 1] + forces_out[i,2] = forcesFlat_out[i*3 + 2] + + bornch_out[i,0] = bornchFlat_out[i*9 + 0] + bornch_out[i,1] = bornchFlat_out[i*9 + 1] + bornch_out[i,2] = bornchFlat_out[i*9 + 2] + + bornch_out[i,3] = bornchFlat_out[i*9 + 3] + bornch_out[i,4] = bornchFlat_out[i*9 + 4] + bornch_out[i,5] = bornchFlat_out[i*9 + 5] + + bornch_out[i,6] = bornchFlat_out[i*9 + 6] + bornch_out[i,7] = bornchFlat_out[i*9 + 7] + bornch_out[i,8] = bornchFlat_out[i*9 + 8] + + + return err, charges_out, forces_out, dipole_out, bornch_out + + + diff --git a/examples/gpmdk/src/gpmd_secuential.F90 b/examples/gpmdk/src/gpmd_secuential.F90 new file mode 100644 index 00000000..9cfac1a2 --- /dev/null +++ b/examples/gpmdk/src/gpmd_secuential.F90 @@ -0,0 +1,69 @@ +!Small driver to call the gpmd main module used +!for compiling the code as a library. + +program gpmd + ! Local modules + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_prepareMD_mod + use gpmdcov_mdloop_mod + use gpmdcov_init_mod + use gpmdcov_part_mod + use gpmdcov_assert_mod + use gpmdcov_mod + use gpmdcov_diagonalize_mod + use gpmdcov_kernel_mod + use gpmdcov_lib_mod + ! Progress modules + use prg_ptable_mod + use prg_system_mod + + implicit none + real(dp), allocatable :: coords_in(:,:) + integer, allocatable :: atomTypes_in(:) + integer, allocatable :: atomic_numbers_in(:) + real(dp), allocatable :: lattice_vectors_in(:,:) + real(dp), allocatable :: forces_out(:,:) + real(dp), allocatable :: charges_out(:) + real(dp), allocatable :: borncharges_out(:,:) + real(dp), allocatable :: dipole_out(:) + type(system_type) :: mysys + integer :: verb_in + + call prg_parse_system(mysys,"benzene-CN.pdb") + allocate(coords_in(3,mysys%nats)) + allocate(atomTypes_in(mysys%nsp)) + allocate(atomic_numbers_in(mysys%nsp)) + allocate(lattice_vectors_in(3,3)) + allocate(charges_out(mysys%nats)) + allocate(forces_out(3,mysys%nats)) + coords_in = mysys%coordinate + atomTypes_in = mysys%spindex + atomic_numbers_in = mysys%spatnum + lattice_vectors_in = mysys%lattice_vector + verb_in = 3 + call gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + &charges_out,forces_out,dipole_out,borncharges_out,verb_in) + + call prg_parse_system(mysys,"benzene-CN.pdb") + deallocate(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in) + allocate(coords_in(3,mysys%nats)) + allocate(atomTypes_in(mysys%nsp)) + allocate(atomic_numbers_in(mysys%nsp)) + allocate(lattice_vectors_in(3,3)) + deallocate(charges_out) + deallocate(forces_out) + allocate(charges_out(mysys%nats)) + allocate(forces_out(3,mysys%nats)) + coords_in = mysys%coordinate + atomTypes_in = mysys%spindex + atomic_numbers_in = mysys%spatnum + lattice_vectors_in = mysys%lattice_vector + verb_in = 1 + call gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + &charges_out,forces_out,dipole_out,borncharges_out,verb_in) + + +end program gpmd + diff --git a/examples/gpmdk/src/gpmdcov_allocation.F90 b/examples/gpmdk/src/gpmdcov_allocation.F90 new file mode 100644 index 00000000..5a4138b0 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_allocation.F90 @@ -0,0 +1,143 @@ +module gpmdcov_allocation_mod + + use bml + + implicit none + + private + + integer, parameter :: dp = kind(1.0d0) + + public :: gpmdcov_reallocate_intVect + public :: gpmdcov_reallocate_realVect + public :: gpmdcov_reallocate_charVect + public :: gpmdcov_reallocate_realMat + public :: gpmdcov_reallocate_denseBmlRealMat + public :: gpmdcov_vect2MatInt + public :: gpmdcov_mat2VectInt + +contains + + !> To reallocate a vector + !! + subroutine gpmdcov_reallocate_realVect(myVect,mySize) + implicit none + real(dp),allocatable :: myVect(:) + integer :: mySize + + if(allocated(myVect))then + deallocate(myVect) + endif + allocate(myVect(mySize)) + + end subroutine gpmdcov_reallocate_realVect + + !> To reallocate a matrix + !! + subroutine gpmdcov_reallocate_realMat(myMat,myRows,myCols) + implicit none + real(dp),allocatable :: myMat(:,:) + integer :: myRows,myCols + + if(allocated(myMat))then + deallocate(myMat) + endif + allocate(myMat(myRows,myCols)) + + end subroutine gpmdcov_reallocate_realMat + + + !> To reallocate a vector of integers + !! + subroutine gpmdcov_reallocate_intVect(myVect,mySize) + implicit none + integer, allocatable :: myVect(:) + integer :: mySize + + if(allocated(myVect))then + deallocate(myVect) + endif + allocate(myVect(mySize)) + + end subroutine gpmdcov_reallocate_intVect + + + !> To reallocate a vector of integers + !! + subroutine gpmdcov_reallocate_charVect(myVect,mySize) + implicit none + character(*), allocatable :: myVect(:) + integer :: mySize + + if(allocated(myVect))then + deallocate(myVect) + endif + allocate(myVect(mySize)) + + end subroutine gpmdcov_reallocate_charVect + + + !> To reallocate a bml matrix + !! + subroutine gpmdcov_reallocate_denseBmlRealMat(myMat,mySize) + + implicit none + integer :: mySize + type(bml_matrix_t) :: myMat + + if(bml_allocated(myMat))then + call bml_deallocate(myMat) + endif + call bml_zero_matrix("dense",bml_element_real,dp,mySize,mySize,myMat) + + end subroutine gpmdcov_reallocate_denseBmlRealMat + + !> To reformat an integer matrix into a vector + !! + subroutine gpmdcov_mat2VectInt(myMat,myVect,rows,cols) + implicit none + integer, allocatable :: myVect(:) + integer, intent(in) :: myMat(:,:) + integer, intent(in) :: cols,rows + integer :: i,j + + if(allocated(myVect))then + deallocate(myVect) + endif + allocate(myVect(cols*rows)) + + do i=1,rows + do j=1,cols + myVect(j + (i-1)*cols) = myMat(j,i) + enddo + enddo + + end subroutine gpmdcov_mat2VectInt + + !> To reformat an integer vector into a matrix + !! + subroutine gpmdcov_vect2MatInt(myVect,myMat,rows,cols) + implicit none + integer, allocatable :: myMat(:,:) + integer, intent(in) :: myVect(:) + integer, intent(in) :: cols,rows + integer :: i,j + + if(allocated(myMat))then + deallocate(myMat) + endif + allocate(myMat(cols,rows)) + + do i=1,rows + do j=1,cols + myMat(j,i) = myVect(j + (i-1)*cols) + enddo + enddo + + end subroutine gpmdcov_vect2MatInt + + + + + +end module gpmdcov_allocation_mod diff --git a/examples/gpmdk/src/gpmdcov_assert.F90 b/examples/gpmdk/src/gpmdcov_assert.F90 new file mode 100644 index 00000000..5b8ee246 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_assert.F90 @@ -0,0 +1,54 @@ +module gpmdcov_assert_mod + + use prg_extras_mod + use latteparser_latte_mod + +contains + + subroutine gpmdcov_assert_input(rank) + use gpmdcov_vars + integer, intent(in) :: rank + + if(rank == 1)then + if(lt%method == "Diag" .and. lt%mucalctype == "FromParts")then + write(*,*)"" + write(*,*)"ERROR: Incompatible parameters set" + write(*,*)"Keyword Method= Diag is incompatible with MuCalcType= FromParts" + write(*,*)"Set Method= DiagEf instead" + write(*,*)"" + stop + endif + + if(lt%method == "Diag" .and. lt%mucalctype == "Combined")then + write(*,*)"" + write(*,*)"ERROR: Incompatible parameters set" + write(*,*)"Keyword Method= Diag is incompatible with MuCalcType= Combined" + write(*,*)"Set Method= DiagEf instead" + write(*,*)"" + stop + endif + + if(lt%method == "Diag" .and. lt%entropy .eqv. .true.)then + write(*,*)"" + write(*,*)"ERROR: Incompatible parameters set" + write(*,*)"Keyword Method= Diag is incompatible with Entropy= T" + write(*,*)"Set Method= DiagEf instead" + write(*,*)"" + stop + endif + + if((lt%dokernel .eqv. .true.) .and. (lt%mucalctype == "Dyn"))then + write(*,*)"" + write(*,*)"ERROR: Incompatible parameters set" + write(*,*)"Keyword DoKernel = T is incompatible with MuCalcType= Dyn" + write(*,*)"Set MuCalcType= FromParts instead" + write(*,*)"" + stop + endif + + + endif + + end subroutine gpmdcov_assert_input + +end module gpmdcov_assert_mod diff --git a/examples/gpmdk/src/gpmdcov_buildz.F90 b/examples/gpmdk/src/gpmdcov_buildz.F90 new file mode 100644 index 00000000..c4747e73 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_buildz.F90 @@ -0,0 +1,25 @@ + + subroutine gpmdcov_buildz(overin_bml,zmatout_bml) + use gpmdcov_vars + + type(bml_matrix_t), intent(inout) :: zmatout_bml + type(bml_matrix_t), intent(inout) :: overin_bml + + igenz = igenz + 1 + + if(lt%zmat == "ZSP")then !Congruence transformation. + + call prg_buildzsparse(overin_bml,zmatout_bml,igenz,lt%mdim,& + lt%bml_type, zk1_bml,zk2_bml,zk3_bml& + ,zk4_bml,zk5_bml,zk6_bml,zsp%nfirst,zsp%nrefi,zsp%nreff,& + zsp%numthresi,zsp%numthresf,zsp%integration,zsp%verbose) + + else + + !Build Z matrix using diagonalization (usual method). + call prg_buildzdiag(overin_bml,zmatout_bml,lt%threshold,lt%mdim,lt%bml_type) + + endif + + end subroutine gpmdcov_buildz + diff --git a/examples/gpmdk/src/gpmdcov_constraints.F90 b/examples/gpmdk/src/gpmdcov_constraints.F90 new file mode 100644 index 00000000..8eec5a1a --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_constraints.F90 @@ -0,0 +1,147 @@ +!> Apply constraints to MD between two atoms +!! Constraints could be potentials, Langrange multipliers, etc. +!! \author Rae Corrigan Grove + +module gpmdcov_constraints_mod + +contains + + !> Apply a switched harmonic potential to constrain two atoms + !! + !! Harmonic to Linear Energy Function + !! \param atom1r Location of atom 1 {x,y,z} + !! \param atom2r Location of atom 2 {x,y,z} + !! \return force_total The collected force for the two atoms being steered will be updated + !! \return energy_total The total energy of steering the two atoms together - added to full potential energy total + subroutine gpmdcov_constraint_harmonicToLinear(atom1r, atom2r, force_total, energy_total, verbose) + + use gpmdcov_writeout_mod + use gpmdcov_vars + implicit none + + real(dp), intent(in) :: atom1r(:), atom2r(:) + real(dp), intent(out) :: energy_total + real(dp), intent(out) :: force_total(:) + real(dp), allocatable :: coords(:,:) + real(dp) :: dcoords(3) + real(dp), allocatable :: Lbox(:) + real(dp) :: r_separation + real(dp) :: switch_low, switch_high, dswitch_low, dswitch_high + real(dp) :: energy_start, energy_end + real(dp) :: denergy_start, denergy_end, force_radial + ! real(dp) :: Lx, Ly, Lz + integer :: k, natms + integer, optional, intent(in) :: verbose + + allocate(coords(2,3)) + allocate(Lbox(3)) + + do k = 1,3 + coords(1,k) = atom1r(k) + coords(2,k) = atom2r(k) + Lbox(k) = sy%lattice_vector(k,k) + enddo + + !> Define separation distance between atoms 1 and 2 + !! Convert to fractional coordinates by dividing by box dimensions + !! Compute fractional distances + do k = 1,3 + dcoords(k) = modulo((atom1r(k) - atom2r(k)) + 0.5_dp*Lbox(k),Lbox(k)) - 0.5_dp * Lbox(k) + enddo + r_separation = norm2(dcoords) + + call gpmdcov_msI("gpmdcov_constraints","Separation distance between & + & steered atoms, r_separation "//to_string(r_separation),lt%verbose,myRank) + + !> Call switching function + call switch_fermi(r_separation, gpmdt%smdr0, switch_low, switch_high, dswitch_low, dswitch_high) + + ! Energy at the start of the switching function (currently harmonic) + energy_start = 0.5d0 * gpmdt%smdforceconststart * (r_separation - gpmdt%smdr0)**2 + ! Energy at the end of the switching function (currently linear) + energy_end = gpmdt%smdforceconstend * r_separation + ! Smoothly switched energy + energy_total = switch_low * energy_start + switch_high * energy_end + + ! Force - negative derivative of energy_start + ! Derivative of energy_end will be the input force constant since energy_end is linear in R + denergy_start = gpmdt%smdforceconststart * (r_separation - gpmdt%smdr0) + + denergy_end = gpmdt%smdforceconstend + + ! Total force across switching function, use chain rule + force_radial = - dswitch_low * energy_start - switch_low * denergy_start - dswitch_high * energy_end & + - switch_high * denergy_end + + !> Determine derivatives wrt x, y, and z directions - multiply by force magnitude to get + !! forces in x, y, and z directions + + force_total(:) = dcoords(:)/r_separation*force_radial + + + deallocate(coords) + + end subroutine gpmdcov_constraint_harmonicToLinear + + !> Polynomial switching function + !! + !! \param r_separation Separation distance between steered atoms + !! \param minsep Minimum separation distance - steering force applied above this distance + !! \param maxsep Maximum separation distance - steering force applied below this distance + !! \return sl Low side of switching function + !! \return sh High side of switching function + !! \return dsl Derivative of low side of switching function + !! \return dsh Derivative of high side of switching function + subroutine switch_polynomial_1(r_separation, minsep, maxsep, sl, sh, dsl, dsh) + use gpmdcov_vars + implicit none + real(dp), intent(in) :: r_separation, minsep, maxsep + real(dp), intent(out) :: sl, sh, dsl, dsh + real(dp) :: x, t + + !> If the separation distance, r_separation, is within the constrained distance range, + !! x gets set to a normalized value between 0 and 1 + !! If the separation distance is outside of the constrained distance range, x is set + !! to either 0 (when the atoms are closer together than the minimum separation distance) + !! or 1 (when the atoms are further apart than the maximum separation distance) + if (r_separation .gt. minsep .AND. r_separation .lt. maxsep) then + x = (r_separation - minsep)/(maxsep - minsep) + else if (r_separation .lt. minsep) then + x = 0.0d0 + else if (r_separation .gt. maxsep) then + x = 1.0d0 + endif + + t = 1.0d0 - x + sl = 3.0d0*t**2.0d0 - 2.0d0*t**3.0d0 + dsl = (6.0d0*t - 6.0d0*t**2.0d0) * (-1.0d0)/(maxsep - minsep) + t = x + sh = 3.0d0*t**2.0d0 - 2.0d0*t**3.0d0 + dsh = (6.0d0*t - 6.0d0*t**2.0d0) * 1.0d0/(maxsep - minsep) + + end subroutine switch_polynomial_1 + + !> Fermi switching function + !! + !! \param r Separation distance between steered atoms + !! \parma r0 Optimal separation distance between steered atoms - center of switching function + !! \return sl Low side of switching function + !! \return sh High side of switching function + !! \return dsl Derivative of low side of switching function + !! \return dsh Derivative of high side of switching function + subroutine switch_fermi(r, r0, sl, sh, dsl, dsh) + use gpmdcov_vars + implicit none + real(dp), intent(in) :: r, r0 + real(dp), intent(out) :: sl, sh, dsl, dsh + real(dp) :: x + + x = r - r0 + sl = 1.0d0 - exp(x)/(1.0d0 + exp(x)) + dsl = exp(2.0d0*x)/(1.0d0 + exp(x))**2 - exp(x)/(1.0d0+exp(x)) + sh = exp(x)/(1.0d0 + exp(x)) + dsh = -1.0d0 * (exp(2.0d0*x)/(1.0d0+exp(x))**2) + exp(x)/(1.0d0+exp(x)) + + end subroutine switch_fermi + +end module gpmdcov_constraints_mod diff --git a/examples/gpmdk/src/gpmdcov_diagonalize.F90 b/examples/gpmdk/src/gpmdcov_diagonalize.F90 new file mode 100644 index 00000000..09ec1429 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_diagonalize.F90 @@ -0,0 +1,158 @@ +!> Diagonalization in parallel with subgraphs. +!! +module gpmdcov_diagonalize_mod + use gpmdcov_vars + use gpmdcov_mod + use gpmdcov_writeout_mod + use prg_extras_mod + +contains + + subroutine gpmdcov_Diagonalize_H0 + implicit none + real(dp) :: mlsi + + if(myRank == 1) mlsi = mls() + call gpmdcov_msMem("gpmdcov_Diagonalize_H0","Before gpmd_Diagonalize_H0 ",lt%verbose,myRank) + +#ifdef DO_MPI + if(allocated(norbsInEachCHAtRank))deallocate(norbsInEachCHAtRank) + allocate(norbsInEachCHAtRank(partsInEachRank(myRank))) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + if(allocated(norbsInEachCHAtRank))deallocate(norbsInEachCHAtRank) + allocate(norbsInEachCHAtRank(gpat%TotalParts)) + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + + norb = syprt(ipt)%estr%norbs + + !> Initialize the orthogonal versions of ham and rho. + if(bml_allocated(syprt(ipt)%estr%oham))call bml_deallocate(syprt(ipt)%estr%oham) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%oham) + + !> Orthogonalize ham. + call prg_orthogonalize(syprt(ipt)%estr%ham0,syprt(ipt)%estr%zmat,syprt(ipt)%estr%oham,& + lt%threshold,lt%bml_type,0) + + !> Allocation + if(allocated(syprt(ipt)%estr%evals))deallocate(syprt(ipt)%estr%evals) + if(allocated(syprt(ipt)%estr%dvals))deallocate(syprt(ipt)%estr%dvals) + if(bml_allocated(syprt(ipt)%estr%evects)) call bml_deallocate(syprt(ipt)%estr%evects) + + allocate(syprt(ipt)%estr%evals(norb)) + allocate(syprt(ipt)%estr%dvals(norb)) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%evects) + + !> Getting eivenvectors and eigenvalues for Core+Halos + call prg_get_evalsDvalsEvects(syprt(ipt)%estr%oham, lt%threshold, syprt(ipt)%estr%hindex, & + & gpat%sgraph(ipt)%llsize, syprt(ipt)%estr%evals, syprt(ipt)%estr%dvals, & + & syprt(ipt)%estr%evects, lt%verbose) + + if(lt%verbose >= 4 .and. myRank == 1) call bml_print_matrix("ham",syprt(ipt)%estr%oham,0,10,0,10) + + norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) + + call bml_deallocate(syprt(ipt)%estr%oham) + + call gpmdcov_msVectRel("Evals for part "//to_string(ipt),syprt(ipt)%estr%evals,lt%verbose,4,myRank) + + enddo + + call gpmdcov_msVectInt("norbsInEachCHAtRank ",norbsInEachCHAtRank,lt%verbose,2,myRank) + + call gpmdcov_msMem("gpmdcov_Diagonalize_H0", "After gpmd_Diagonalize_H0",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Diagonalize_H0", "Time for gpmd_Diagonalize_H0 "//to_string(mls() - mlsi),lt%verbose,myRank) + + end subroutine gpmdcov_Diagonalize_H0 + + + subroutine gpmdcov_Diagonalize_H1(nguess) + real(dp), allocatable :: nguess(:) + real(dp) :: mlsi + + mlsi = mls() +#ifdef DO_MPI + if(allocated(norbsInEachCHAtRank)) deallocate(norbsInEachCHAtRank) + allocate(norbsInEachCHAtRank(partsInEachRank(myRank))) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + if(allocated(norbsInEachCHAtRank)) deallocate(norbsInEachCHAtRank) + allocate(norbsInEachCHAtRank(gpat%TotalParts)) + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + norb = syprt(ipt)%estr%norbs + + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before pot alloc ",lt%verbose,myRank) + if(allocated(syprt(ipt)%estr%coul_pot_k))deallocate(syprt(ipt)%estr%coul_pot_k) + if(allocated(syprt(ipt)%estr%coul_pot_r))deallocate(syprt(ipt)%estr%coul_pot_r) + allocate(syprt(ipt)%estr%coul_pot_k(syprt(ipt)%nats)) + allocate(syprt(ipt)%estr%coul_pot_r(syprt(ipt)%nats)) + + syprt(ipt)%estr%coul_pot_k = 0.0_dp + syprt(ipt)%estr%coul_pot_r = 0.0_dp + syprt(ipt)%net_charge = 0.0_dp + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After pot alloc ",lt%verbose,myRank) + + !> Get Coulombic potential and charges for the part. + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before pot asign ",lt%verbose,myRank) + do j=1,gpat%sgraph(ipt)%lsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + syprt(ipt)%estr%coul_pot_k(j) = coul_pot_k(jj) + syprt(ipt)%estr%coul_pot_r(j) = coul_pot_r(jj) + syprt(ipt)%net_charge(j) = nguess(jj) + enddo + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After pot asign ",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before hscf ",lt%verbose,myRank) + if(bml_allocated(syprt(ipt)%estr%ham)) call bml_deallocate(syprt(ipt)%estr%ham) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%ham) + !call bml_copy_new(syprt(ipt)%estr%ham0,syprt(ipt)%estr%ham) + call bml_copy(syprt(ipt)%estr%ham0,syprt(ipt)%estr%ham) + !> Get the scf hamiltonian. The output is ham_bml. + call gpmdcov_msIII("gpmdcov_DM_Min","In prg_get_hscf...",lt%verbose,myRank) + call prg_get_hscf(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%estr%ham,syprt(ipt)%spindex,& + &syprt(ipt)%estr%hindex,tb%hubbardu,syprt(ipt)%net_charge,& + &syprt(ipt)%estr%coul_pot_r,syprt(ipt)%estr%coul_pot_k,lt%mdim,lt%threshold) + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After hscf ",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before orth ",lt%verbose,myRank) + if(bml_allocated(syprt(ipt)%estr%oham)) call bml_deallocate(syprt(ipt)%estr%oham) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%oham) + + !> Orthogonalize ham. + call prg_orthogonalize(syprt(ipt)%estr%ham,syprt(ipt)%estr%zmat,syprt(ipt)%estr%oham,& + <%threshold,lt%bml_type,0) + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After orth",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before evals realloc",lt%verbose,myRank) + if(allocated(syprt(ipt)%estr%evals)) deallocate(syprt(ipt)%estr%evals) + if(allocated(syprt(ipt)%estr%dvals)) deallocate(syprt(ipt)%estr%dvals) + if(bml_allocated(syprt(ipt)%estr%evects)) call bml_deallocate(syprt(ipt)%estr%evects) + + allocate(syprt(ipt)%estr%evals(norb)) + allocate(syprt(ipt)%estr%dvals(norb)) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%evects) + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After evals realloc",lt%verbose,myRank) + + !> Diagonalize ham + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before diag",lt%verbose,myRank) + call prg_get_evalsDvalsEvects(syprt(ipt)%estr%oham, lt%threshold, syprt(ipt)%estr%hindex, & + & gpat%sgraph(ipt)%llsize, syprt(ipt)%estr%evals, syprt(ipt)%estr%dvals, & + & syprt(ipt)%estr%evects, lt%verbose) + + norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) + + call bml_deallocate(syprt(ipt)%estr%oham) + call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "After diag",lt%verbose,myRank) + + enddo + call gpmdcov_msII("gpmdcov_Diagonalize_H1", "Time for gpmd_Diagonalize_H1 "//to_string(mls() - mlsi),lt%verbose,myRank) + + end subroutine gpmdcov_Diagonalize_H1 + +end module gpmdcov_Diagonalize_mod diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 new file mode 100644 index 00000000..a7b077bf --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -0,0 +1,600 @@ +module gpmdcov_DM_Min_mod + use gpmdcov_nonequilibrium_mod + +contains + + !> SCF loop + !! + subroutine gpmdcov_DM_Min(Nr_SCF,nguess,mix) + + use gpmdcov_vars + use gpmdcov_mod + use gpmdcov_rhosolver_mod + use gpmdcov_writeout_mod + use gpmdcov_kernel_mod + use gpmdcov_diagonalize_mod + use gpmdcov_field + + integer, intent(in) :: Nr_SCF + real(dp), allocatable :: nguess(:) + logical, intent(in) :: mix + real(dp) :: tch1, mls_coul + + converged = .false. + charges_old = nguess + + call gpmdcov_msMem("gpmdcov_DM_Min","Berofe gpmd_DM_Min",lt%verbose,myRank) + + if(.not.allocated(auxcharge))allocate(auxcharge(sy%nats)) + + ! Beginning of the SCF loop. + do iscf=1,Nr_SCF + + if (myRank == 1) write(*,*)"SCF iter", iscf + + call gpmdcov_msI("gpmdcov_DM_Min","rank "//to_string(myRank)//" SCF iter "//to_string(iscf),lt%verbose,myRank) + + !> Real contribution to the Coulomb energy. The outputs are coul_forces_r,coul_pot_r. + call gpmdcov_msII("gpmdcov_DM_Min","In real Coul ...",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_coul = mls() + ! call get_ewald_list_real(sy%spindex,sy%splist,sy%coordinate& + ! ,nguess,tb%hubbardu,sy%lattice_vector,& + ! sy%volr,lt%coul_acc,lt%timeratio,nl%nnRx,nl%nnRy,& + ! nl%nnRz,nl%nrnnlist,nl%nnType,coul_forces_r,coul_pot_r); + + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,nguess,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,coul_forces_r,coul_pot_r); + call gpmdcov_msII("gpmdcov_DM_Min","Time real coul "//to_string(mls() - mls_coul)//" ms",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) write(*,*)"Time for real coul",mls() - mls_coul + + + !> Reciprocal contribution to the Coul energy. The outputs are coul_forces_k,coul_pot_k. + call gpmdcov_msII("gpmdcov_DM_Min","In recip Coul ...",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_coul = mls() + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + ,nguess,tb%hubbardu,sy%lattice_vector,& + sy%recip_vector,sy%volr,lt%coul_acc,coul_forces_k,coul_pot_k); + call gpmdcov_msII("gpmdcov_DM_Min","Time recip coul "//to_string(mls() - mls_coul)//" ms",lt%verbose,myRank) + + if(iscf == Nr_SCF) converged = .true. + +!!!!!!!!!!!!!!!!!!!!!!!!!!! + !Loop over parts +!!!!!!!!!!!!!!!!!!!!!!!!!!! + + auxcharge = 0.0_dp + mls_i = mls() +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + norb = syprt(ipt)%estr%norbs + + if(.not.allocated(syprt(ipt)%estr%coul_pot_k))then + allocate(syprt(ipt)%estr%coul_pot_k(syprt(ipt)%nats)) + allocate(syprt(ipt)%estr%coul_pot_r(syprt(ipt)%nats)) + endif + + syprt(ipt)%estr%coul_pot_k = 0.0_dp + syprt(ipt)%estr%coul_pot_r = 0.0_dp + syprt(ipt)%net_charge = 0.0_dp + + !> Get Coulombic potential and charges for the part. + do j=1,gpat%sgraph(ipt)%lsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + syprt(ipt)%estr%coul_pot_k(j) = coul_pot_k(jj) + syprt(ipt)%estr%coul_pot_r(j) = coul_pot_r(jj) + syprt(ipt)%net_charge(j) = nguess(jj) + enddo + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%ham) + call bml_copy_new(syprt(ipt)%estr%ham0,syprt(ipt)%estr%ham) + + !> Get the scf hamiltonian. The output is ham_bml. + call gpmdcov_msIII("gpmdcov_DM_Min","In prg_get_hscf ...",lt%verbose,myRank) + call prg_get_hscf(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%estr%ham,syprt(ipt)%spindex,& + syprt(ipt)%estr%hindex,tb%hubbardu,syprt(ipt)%net_charge,& + syprt(ipt)%estr%coul_pot_r,syprt(ipt)%estr%coul_pot_k,lt%mdim,lt%threshold) + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%oham) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%orho) + + !> Orthogonalize ham. + call prg_orthogonalize(syprt(ipt)%estr%ham,syprt(ipt)%estr%zmat,syprt(ipt)%estr%oham,& + lt%threshold,lt%bml_type,0) + + !> Now solve for the desity matrix. + call gpmdcov_RhoSolver(syprt(ipt)%estr%oham,syprt(ipt)%estr%orho,syprt(ipt)%estr%evects) + norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) + + !call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,rho_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%rho) + + !> Deprg_orthogonalize orthop_bml to get the density matrix rho_bml. + call gpmdcov_msIII("gpmdcov_DM_Min","Entering prg_deorthogonalize...",lt%verbose,myRank) + call prg_deorthogonalize(syprt(ipt)%estr%orho,syprt(ipt)%estr%zmat,syprt(ipt)%estr%rho,& + lt%threshold,lt%bml_type,0) + call gpmdcov_msIII("gpmdcov_DM_Min","Leaving prg_deorthogonalize...",lt%verbose,myRank) + + !> Get the system charges from rho + call gpmdcov_msIII("gpmdcov_DM_Min","Getting system charges...",lt%verbose,myRank) + call prg_get_charges(syprt(ipt)%estr%rho,syprt(ipt)%estr%over,syprt(ipt)%estr%hindex,syprt(ipt)%net_charge,tb%numel,& + syprt(ipt)%spindex,lt%mdim,lt%threshold) + call gpmdcov_msIII("gpmdcov_DM_Min","Leaving Get charges...",lt%verbose,myRank) + + call gpmdcov_msIII("gpmdcov_DM_Min","Total charge of the part for Rank "//to_string(printRank())// & + & "="//to_string(sum(syprt(ipt)%net_charge(:)))// & + & "("//to_string(size(syprt(ipt)%net_charge,dim=1))//")",lt%verbose,myRank) + + call gpmdcov_msIII("gpmdcov_DM_Min","Total charge of the core part for Rank "//to_string(printRank())//& + & "="//to_string(sum(syprt(ipt)%net_charge(1:gpat%sgraph(ipt)%llsize))),lt%verbose,myRank) + + do ii=1,gpat%sgraph(ipt)%llsize + j = gpat%sgraph(ipt)%core_halo_index(ii)+1 + auxcharge(j) = syprt(ipt)%net_charge(ii) + enddo + + enddo + + call gpmdcov_msII("gpmdcov_DM_Min","Time for get qs of all parts "//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + + call prg_wait() + + mls_i = mls() + + +#ifdef DO_MPI + if (getNRanks() > 1) then + call prg_sumRealReduceN(auxcharge(:), sy%nats) + endif +#endif + call gpmdcov_msII("gpmdcov_DM_Min","MPI rank finished prg_sumRealReduceN for qs "& + &//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + + nguess = auxcharge + + scferror = norm2(nguess(:)-charges_old(:))/sqrt(real(sy%nats,dp)) + if(mix)then + if(kernel%kernelMixing .and. lt%dokernel)then + if( scferror < 0.1_dp)then + if(firstKernel)then + call gpmdcov_getKernel(sy%nats) + KSum = 0.0_dp + endif + nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Kernel assisted mixing",lt%verbose,myRank) + else + nguess = charges_old + 0.1_dp*(nguess-charges_old) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Kernel assisted mixing (linear part)",lt%verbose,myRank) + endif + else + call prg_qmixer(nguess,charges_old,dqin,& + dqout,scferror,iscf,lt%pulaycoeff,lt%mpulay,0) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Pulay/DIIS mixing ",lt%verbose,myRank) + endif + else + call prg_linearmixer(nguess,charges_old,scferror,lt%mixcoeff,lt%verbose) + endif + + if(myRank == 1) write(*,*)"gpmdcov_DM_Min","SCF Error "//to_string(scferror) + + tch1 = sum(charges_old) + charges_old = nguess + tch = sum(nguess) + + call gpmdcov_msII("gpmdcov_dm_min", "Total charge ="//to_string(tch),lt%verbose,myRank) + + !Get the chemical potential. Feb 2021 implemetation + ! if(lt%MuCalcType == "Dyn")then + ! call gpmdcov_muDyn(nguess,Nr_SCF) + ! elseif(lt%MuCalcType == "FromParts")then + ! call gpmdcov_muFromParts() + ! elseif(lt%MuCalcType == "Combined")then + ! call gpmdcov_muFromParts() + ! call gpmdcov_muDyn(nguess,Nr_SCF) + ! else + ! call gpmdcov_msI("gpmdcov_getmu","No Mu Calculation method. I will use & + ! & a fixed mu instead ...",lt%verbose,myRank) + ! endif + + if(converged)then ! To do a last extra step. + exit + else + if(scferror < lt%scftol .and. iscf > 1) then + if (myRank == 1) then + write(*,*)""; write(*,*)"SCF converged within",iscf,"steps ..." + write(*,*)"SCF error =",scferror + endif + converged = .true. + endif + endif + enddo + + call gpmdcov_msI("gpmdcov_dm_min", "Total charge ="//to_string(tch),lt%verbose,myRank) + + !> End of SCF loop. + if (lt%verbose == 6 .and. converged)then + ipt = 1 + bndfil = real(0.5*nel/norb) + allocate(eigenvalues(norb)) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,aux_bml) + call bml_write_matrix(syprt(ipt)%estr%ham,"H.mtx") + + call prg_build_density_T0(syprt(ipt)%estr%oham, aux_bml, lt%threshold, bndfil, eigenvalues) + call bml_write_matrix(aux_bml,"D.mtx") + + sparsity = bml_get_sparsity(aux_bml, 1.0D-5) + write(*,*)"Sparsity Rho=",sparsity + + !Getting the fermi level + ef = (eigenvalues(int(bndfil*norb)+1) + eigenvalues(int(bndfil*norb)))/2 + !eigenvalues = eigenvalues - ef + write(*,*)"HOMO=",eigenvalues(int(norb/2)) + write(*,*)"LUMO=",eigenvalues(int(norb/2)+1) + write(*,*)"Ef=", ef, int(bndfil*norb), norb + !Writting the total DOS + !call prg_write_tdos(eigenvalues, 0.05d0, 10000, -20.0d0, 20.0d0, "tdos.dat") + + deallocate(eigenvalues) + call bml_deallocate(aux_bml) + stop + + endif + + deallocate(auxcharge) + + if(myRank == 1 .and. lt%verbose >= 3)then + write(*,*)"" + write(*,*)"Total charge of the system (After SCF) =",sum(nguess(:),dim=1),size(nguess(:),dim=1) + write(*,*)"" + call prg_write_system(sy,"charged_system","pdb") + write(*,*)""; write(*,*)"System charges (After SCF):" + do j=1,sy%nats + write(*,*)sy%symbol(j),nguess(j) + enddo + endif + + call gpmdcov_msMem("gpmdcov_dm_min", "After gpmd_DM_Min",lt%verbose,myRank) + + end subroutine gpmdcov_DM_Min + + + subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) + + use gpmdcov_vars + use gpmdcov_mod + use gpmdcov_rhosolver_mod + use gpmdcov_writeout_mod + use gpmdcov_kernel_mod + use gpmdcov_Diagonalize_mod + + integer, intent(in) :: Nr_SCF + real(dp), allocatable :: nguess(:),kernelTimesRes(:) + logical, intent(in) :: mix,applyField + real(dp) :: tch1 + real(8) :: mls_v, mls_coul, mls_mu, mls_red, mls_mix, mls_scfIter + real(dp), allocatable :: KK0Res(:) + + converged = .false. + if(.not.allocated(charges_old))allocate(charges_old(sy%nats)) + charges_old = nguess + + call gpmdcov_msMem("gpmdcov_dm_min_eig","Before gpmd_DM_Min_Eig",lt%verbose,myRank) + + ! Beginning of the SCF loop. + if(.not.allocated(auxcharge))allocate(auxcharge(sy%nats)) + + do iscf=1,Nr_SCF + + mls_scfIter = mls() + + call gpmdcov_msI("gpmdcov_DM_Min","rank "//to_string(myRank)//" SCF iter"//to_string(iscf),lt%verbose,myRank) + + !> Real contribution to the Coul energy. The outputs are + !coul_forces_r,coul_pot_r. + call gpmdcov_msI("gpmdcov_DM_Min","In real Coul ...",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_dm_min_eig","Before get_ewald_list_real_dcalc_vect",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_coul = mls() + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,nguess,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,coul_forces_r,coul_pot_r); + call gpmdcov_msII("gpmdcov_DM_Min","Time real coul "//to_string(mls() - mls_coul)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_dm_min_eig","After get_ewald_list_real_dcalc_vect",lt%verbose,myRank) + + !> Reciprocal contribution to the Coul energy. The outputs are + !coul_forces_k,coul_pot_k. + call gpmdcov_msI("gpmdcov_DM_Min","In recip Coul ...",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_dm_min_eig","Before get_ewald_recip",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_coul = mls() + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + &,nguess,tb%hubbardu,sy%lattice_vector,& + &sy%recip_vector,sy%volr,lt%coul_acc,coul_forces_k,coul_pot_k); + call gpmdcov_msII("gpmdcov_DM_Min","Time recip coul "//to_string(mls() - mls_coul)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_dm_min_eig","After get_ewald_recip",lt%verbose,myRank) + + if(iscf == Nr_SCF) converged = .true. + + call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before gpmd_diagonalize_H1",lt%verbose,myRank) + call gpmdcov_diagonalize_H1(nguess) + call gpmdcov_msMem("gpmdcov_dm_min_eig", "After gpmd_diagonalize_H1",lt%verbose,myRank) + + if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then + call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before gpmdcov_muFromParts",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_mu = mls() + call gpmdcov_muFromParts() + call gpmdcov_msII("gpmdcov_DM_Min","Time for get Mu "//to_string(mls() - mls_mu)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_dm_min_eig", "After gpmdcov_muFromParts",lt%verbose,myRank) + else + call gpmdcov_msI("gpmdcov_getmu","No Mu Calculation method. I will use & + & a fixed mu instead ...",lt%verbose,myRank) + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!! + !Loop over parts +!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before Loop over parts",lt%verbose,myRank) + auxcharge = 0.0_dp + mls_i = mls() +#ifdef DO_MPI + ! !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + norb = syprt(ipt)%estr%norbs + + if(bml_allocated(syprt(ipt)%estr%orho))call bml_deallocate(syprt(ipt)%estr%orho) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%orho) + + mls_v = mls() + call prg_build_density_fromEvalsAndEvects(syprt(ipt)%estr%evects, syprt(ipt)%estr%evals,& + & syprt(ipt)%estr%orho, lt%threshold, bndfil, lt%kbt, Ef, lt%verbose) + call gpmdcov_msI("prg_build_density_fromEvalsAndEvects","Time build density "& + &//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) + + norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) + + if(bml_allocated(syprt(ipt)%estr%rho))call bml_deallocate(syprt(ipt)%estr%rho) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%rho) + + !> Deprg_orthogonalize orthop_bml to get the density matrix rho_bml. + mls_v = mls() + call gpmdcov_msIII("gpmdcov_DM_Min","Entering prg_deorthogonalize...",lt%verbose,myRank) + call prg_deorthogonalize(syprt(ipt)%estr%orho,syprt(ipt)%estr%zmat,syprt(ipt)%estr%rho,& + lt%threshold,lt%bml_type,0) + call gpmdcov_msIII("gpmdcov_DM_Min","Leaving prg_deorthogonalize...",lt%verbose,myRank) + call gpmdcov_msI("prg_build_density_fromEvalsAndEvects","Time for deorthogonalize "& + &//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) + + !> Get the system charges from rho + call gpmdcov_msIII("gpmdcov_DM_Min","Getting system charges...",lt%verbose,myRank) + mls_v = mls() + call prg_get_charges(syprt(ipt)%estr%rho,syprt(ipt)%estr%over,syprt(ipt)%estr%hindex,syprt(ipt)%net_charge,tb%numel,& + &syprt(ipt)%spindex,lt%mdim,lt%threshold) + call gpmdcov_msIII("gpmdcov_DM_Min","Leaving Get charges...",lt%verbose,myRank) + call gpmdcov_msI("prg_build_density_fromEvalsAndEvects","Time for get charges "& + &//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) + + call gpmdcov_msIII("gpmdcov_DM_Min","Total charge of the part for Rank "//to_string(printRank())// & + & "="//to_string(sum(syprt(ipt)%net_charge(:)))// & + & "("//to_string(size(syprt(ipt)%net_charge,dim=1))//")",lt%verbose,myRank) + + call gpmdcov_msIII("gpmdcov_DM_Min","Total charge of the core part for Rank "//to_string(printRank())//& + & "="//to_string(sum(syprt(ipt)%net_charge(1:gpat%sgraph(ipt)%llsize))),lt%verbose,myRank) + + do ii=1,gpat%sgraph(ipt)%llsize + j = gpat%sgraph(ipt)%core_halo_index(ii)+1 + auxcharge(j) = syprt(ipt)%net_charge(ii) + enddo + + !> Get the system currents from rho and ham + if(gpmdt%compcurr)then + if(MDStep >= 2 .and. (mod(MDstep,gsp2%parteach) .ne. 0))then + norb_core = syprt(ipt)%estr%hindex(2,gpat%sgraph(ipt)%llsize) + call gpmdcov_get_currents(syprt(ipt)%estr%ham,oldsyprt(ipt)%estr%rho,syprt(ipt)%estr%zmat,& + &syprt(ipt)%estr%hindex,gpat%sgraph(ipt)%llsize,norb_core,gpat%sgraph(ipt)%core_halo_index,sy%symbol,& + &gpmdt%currthr) + endif + + !Storing the old subsystem density matrix + if(bml_allocated(oldsyprt(ipt)%estr%rho))then + call bml_deallocate(oldsyprt(ipt)%estr%rho) + endif + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,oldsyprt(ipt)%estr%rho) + call bml_copy(syprt(ipt)%estr%rho, oldsyprt(ipt)%estr%rho) + endif + + enddo + + mls_red = mls() +#ifdef DO_MPI + if (getNRanks() > 1) then + call prg_sumRealReduceN(auxcharge(:), sy%nats) + endif +#endif + call gpmdcov_msMem("gpmdcov_dm_min_eig", "After Loop over parts",lt%verbose,myRank) + call gpmdcov_msIII("gpmdcov_DM_Min","Time for prg_sumRealReduceN& + &for qs "//to_string(mls() - mls_red)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before Kernel logic",lt%verbose,myRank) + nguess = auxcharge + scferror = 0.0d0 + scferror = norm2(nguess(:)-charges_old(:))/sqrt(real(sy%nats,dp)) +! if(myRank == 1) write(*,*)"ResVect", nguess(1:5)-charges_old(1:5) +! if(myRank == 1) write(*,*)"nguess", nguess(1:5) +! if(myRank == 1) write(*,*)"charges_old", charges_old(1:5) +! if(myRank == 1) write(*,*)"SCF Error",iscf,scferror +! if(myRank == 1) write(*,*)"mdstep,FirstKernel,NewPart,scferror,mix",mdstep,firstKernel,newPart,scferror,mix + + mls_mix = mls() + if(mix)then +! if(myRank == 1) write(*,*)mdstep,"InIf (mix)" + if( kernel%kernelMixing .and. lt%dokernel)then +! if(myRank == 1) write(*,*)mdstep,"InIf ( kernel%kernelMixing .and. lt%dokernel)" + if( scferror < 0.01_dp .or. newPart)then +! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart)" + if(firstKernel .or. kernel%buildAlways)then +! if(myRank == 1) write(*,*)mdstep,"InIf (firstKernel .or. kernel%buildAlways)" + if(kernel%kernelType == "Full")then + call gpmdcov_getKernel(sy%nats) + elseif(kernel%kernelType == "ByBlocks")then + call gpmdcov_getKernel_byBlocks(sy%nats) + elseif(kernel%kernelType == "ByParts")then + call gpmdcov_getKernel_byParts(syprt,syprtk) + else + write(*,*)"The TypeOfKernel is not implemented" + stop + endif + + firstKernel = .false. + newPart = .false. + if((kernel%kernelType == "ByParts" ) .and. (.not. kernel%updateAfterBuild))then + allocate(kernelTimesRes(sy%nats)) +! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart), Applying K" + !CHANGE Check with Anders + !if(mdstep <= xl%minit)then + call gpmdcov_applyKernel(nguess,charges_old,syprtk,kernelTimesRes) + nguess = charges_old - kernelTimesRes + !endif + ! call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) + ! nguess = charges_old - KK0Res + !nguess = nguess + !endif + + deallocate(kernelTimesRes) + elseif(kernel%kernelType == "ByParts" .and. kernel%updateAfterBuild)then + call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) + nguess = charges_old - KK0Res + else + nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) + endif + else + !Update and apply + if(myRank == 1) write(*,*)mdstep,"InIf elseOf( scferror < 0.01_dp .or. newPart)" + if(kernel%rankNUpdate > 0)then + if(kernel%kernelType == "Full")then + write(*,*)"The Kernel rank N updates for the KernelType=Full is not implemented" + stop + elseif(kernel%kernelType == "ByParts")then + call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) + nguess = charges_old - KK0Res + endif + else !Just apply the old one + if(kernel%kernelType == "Full")then + nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) + elseif(kernel%kernelType == "ByParts")then + allocate(kernelTimesRes(sy%nats)) + call gpmdcov_applyKernel(nguess,charges_old,syprtk,kernelTimesRes) + nguess = charges_old - kernelTimesRes + deallocate(kernelTimesRes) + endif + endif + + endif + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Kernel assisted mixing",lt%verbose,myRank) + else !if( scferror < 0.01_dp .or. newPart) + if(kernel%initialMixingWith == "Lin")then + nguess = charges_old + lt%pulaycoeff*(nguess-charges_old) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Initial Kernel mixing using linear mixing",lt%verbose,myRank) + else + call prg_qmixer(nguess,charges_old,dqin,& + dqout,scferror,iscf,lt%pulaycoeff,lt%mpulay,0) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Initial Kernel mixing using Pulay/DIIS mixing",lt%verbose,myRank) + endif + endif + else !if( kernel%kernelMixing .and. lt%dokernel) + call prg_qmixer(nguess,charges_old,dqin,& + dqout,scferror,iscf,lt%pulaycoeff,lt%mpulay,0) + call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Pulay/DIIS mixing",lt%verbose,myRank) + endif + else !if(mix) + if(lt%dokernel .and. firstKernel)then + if(kernel%kernelType == "Full")then + call gpmdcov_getKernel(sy%nats) + elseif(kernel%kernelType == "ByBlocks")then + call gpmdcov_getKernel_byBlocks(sy%nats) + elseif(kernel%kernelType == "ByParts")then + call gpmdcov_getKernel_byParts(syprt,syprtk) + else + write(*,*)"The TypeOfKernel is not implemented" + stop + endif + + firstKernel = .false. + !CHANGE + ! if(kernel%kernelType == "ByParts")then + ! allocate(kernel imesRes(sy%nats)) + ! call gpmdcov_applyKernel(nguess,charges_old,syprtk,kernelTimesRes) + ! nguess = charges_old - kernelTimesRes + ! deallocate(kernelTimesRes) + ! else + ! nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) + ! endif + + else + !CHANGE + !call prg_linearmixer(nguess,charges_old,scferror,lt%mixcoeff,lt%verbose) + nguess = nguess + endif + + endif + call gpmdcov_msI("gpmdcov_DM_Min","Time for Mix & + & "//to_string(mls() - mls_mix)//" ms",lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov_dm_min_eig", "After Kernel logic",lt%verbose,myRank) + + if(myRank == 1) write(*,*)"gpmdcov_DM_Min","SCF Error "//to_string(scferror) + + tch1 = sum(charges_old) + charges_old = nguess + tch = sum(nguess) + call prg_wait + if(lt%MuCalcType == "Combined" .or. lt%MuCalcType == "Dyn")then + call gpmdcov_muDyn(nguess,Nr_SCF) + endif + + call gpmdcov_msII("gpmdcov_dm_min", "Total charge="//to_string(tch),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_DM_Min","Time for one SCF iter "& + &//to_string(mls() - mls_scfIter)//" ms",lt%verbose,myRank) + + if(converged)then ! To do a last extra step. + ! if(lt%dokernel)then + ! if(kernel%kernelType == "Full")then + ! call gpmdcov_getKernel(sy%nats) + ! elseif(kernel%kernelType == "ByBlocks")then + ! call gpmdcov_getKernel_byBlocks(sy%nats) + ! elseif(kernel%kernelType == "ByParts")then + ! call gpmdcov_getKernel_byParts(sy%nats,syprt,syprtk) + ! endif + ! endif + + exit + else + if(scferror < lt%scftol .and. iscf > 1) then + if (myRank == 1) then + write(*,*)""; write(*,*)"SCF converged within",iscf,"steps ..." + write(*,*)"SCF error =",scferror + endif + converged = .true. + endif + endif + enddo + newPart = .false. + deallocate(auxcharge) + + call gpmdcov_msMem("gpmdcov_dm_min", "After gpmd_DM_Min",lt%verbose,myRank) + + end subroutine gpmdcov_DM_Min_Eig + + +end module gpmdcov_DM_Min_mod diff --git a/examples/gpmdk/src/gpmdcov_dump.F90 b/examples/gpmdk/src/gpmdcov_dump.F90 new file mode 100644 index 00000000..8590ec77 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_dump.F90 @@ -0,0 +1,38 @@ + + !> Dump GPMD + subroutine gpmdcov_dump() + + use gpmdcov_vars + + if(myRank == 1)then + write(*,*)"Dumping resetart file ..." + + open(1,file='restart.dmp',form="unformatted",access="sequential") + write(1)sy%nats + write(1)sy%symbol + write(1)sy%atomic_number + write(1)sy%coordinate + write(1)sy%velocity + write(1)sy%force + write(1)sy%net_charge + write(1)sy%mass + write(1)sy%spindex + write(1)sy%lattice_vector + write(1)sy%spatnum + write(1)sy%spmass + + !Dump xlbo + write(1)mdstep + write(1)n + write(1)n_0 + write(1)n_1 + write(1)n_2 + write(1)n_3 + write(1)n_4 + write(1)n_5 + + close(1) + endif + + end subroutine gpmdcov_dump + diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 new file mode 100644 index 00000000..004d9866 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -0,0 +1,379 @@ +module gpmdcov_EnergAndForces_mod + + use gpmdcov_mod + + contains + + subroutine gpmdcov_EnergAndForces(charges) + + use gpmdcov_vars + use gpmdcov_writeout_mod + use gpmdcov_kernel_mod + use gpmdcov_constraints_mod + + Implicit none + real(dp), intent(in) :: charges(:) + real(dp), allocatable :: ebandvector(:) + real(dp), allocatable :: R1(:), R2(:) + real(dp) :: dcoords(3), dist + real(dp) :: smd_total_force(3), smd_total_energy + real(dp) :: smd_total_energy_allpairs + real(dp) :: smd_test_force(3), smd_test_energy + real(dp) :: deltas(3) + real(dp) :: delta_h, energy_plus, energy_minus, denergy, differ + integer :: k + logical :: testsmd + + call gpmdcov_msMem("gpmdcov","Before gpmd_EnergAndForces",lt%verbose,myRank) + + if(.not.allocated(coul_forces)) allocate(coul_forces(3,sy%nats)) + if(.not.allocated(GFPUL))allocate(GFPUL(3,sy%nats)) + if(.not.allocated(GFSCOUL))allocate(GFSCOUL(3,sy%nats)) + if(.not.allocated(SKForce))allocate(SKForce(3,sy%nats)) + if(.not.allocated(collectedforce))allocate(collectedforce(3,sy%nats)) + if(.not.allocated(ebandvector))allocate(ebandvector(gpat%TotalParts)) + + GFPUL = 0.0_dp + GFSCOUL = 0.0_dp + SKForce = 0.0_dp + collectedforce = 0.0_dp + ebandvector = 0.0_dp + smd_total_force(:) = 0.0_dp + smd_total_energy = 0.0_dp + smd_total_energy_allpairs = 0.0_dp + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !> Loop over all the parts +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + !Distribute the charges back to the parts. + do j=1,gpat%sgraph(ipt)%lsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + syprt(ipt)%net_charge(j) = charges(jj) + enddo + + norb = syprt(ipt)%estr%norbs + + norb_core = syprt(ipt)%estr%hindex(2,gpat%sgraph(ipt)%llsize) + + if(bml_get_N(aux_bml).gt.0)then + call bml_deallocate(aux_bml) + call bml_deallocate(aux1_bml) + deallocate(row) + endif + + allocate(row(norb)) + + !> Get Electronic energy + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,nOrb,nOrb,aux1_bml) + call bml_copy_new(syprt(ipt)%estr%rho,aux_bml) + + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,nOrb,nOrb,rhoat_bml) + call prg_build_atomic_density(rhoat_bml,tb%numel,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,norb,& + lt%bml_type) + + call bml_add_deprecated(1.0_dp,aux_bml,-1.0_dp,rhoat_bml,lt%threshold) + call bml_multiply(aux_bml,syprt(ipt)%estr%ham,aux1_bml,1.0d0, 0.0d0,lt%threshold) + row=0.0_dp + call bml_deallocate(rhoat_bml) + call bml_get_diagonal(aux1_bml,row) + + TRRHOH = 0.0_dp + do i=1,norb_core + TRRHOH= TRRHOH+ row(i) + enddo + + call gpmdcov_message("gpmdcov_EnergAndForces","Energy Band for part =& + & "//to_string(ipt)//"= "//to_string(TRRHOH),lt%verbose,myRank) + + call bml_deallocate(aux_bml) + call bml_deallocate(aux1_bml) + call bml_deallocate(syprt(ipt)%estr%oham) + deallocate(row) + + syprt(ipt)%estr%eband = TRRHOH + ebandvector(ipt) = TRRHOH + + dx = 0.0001_dp; + + if(gpmdt%usevectsk)then + + call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& + syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + lt%threshold, dH0x_bml,dH0y_bml,dH0z_bml) + + call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& + syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + lt%threshold, dSx_bml,dSy_bml,dSz_bml) + else + call get_dH(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& + syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + lt%threshold, dH0x_bml,dH0y_bml,dH0z_bml) + + call get_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& + syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + lt%threshold, dSx_bml,dSy_bml,dSz_bml) + endif + + if(printRank() == 1 .and. lt%verbose >= 10)then + call bml_print_matrix("dH0x_bml",dH0x_bml,0,10,0,10) + call bml_print_matrix("dH0y_bml",dH0y_bml,0,10,0,10) + call bml_print_matrix("dH0z_bml",dH0z_bml,0,10,0,10) + + call bml_print_matrix("dSx_bml",dSx_bml,0,10,0,10) + call bml_print_matrix("dSy_bml",dSy_bml,0,10,0,10) + call bml_print_matrix("dSz_bml",dSz_bml,0,10,0,10) + endif + + call get_skforce(syprt(ipt)%nats,syprt(ipt)%estr%rho,dH0x_bml,dH0y_bml,& + dH0z_bml,syprt(ipt)%estr%hindex,syprt(ipt)%estr%SKForce,lt%threshold) + + call prg_get_pulayforce(syprt(ipt)%nats,syprt(ipt)%estr%zmat,syprt(ipt)%estr%ham,syprt(ipt)%estr%rho,& + dSx_bml,dSy_bml,dSz_bml,syprt(ipt)%estr%hindex,syprt(ipt)%estr%FPUL,lt%threshold) + + call get_nonortho_coul_forces(syprt(ipt)%nats, norb, dSx_bml,dSy_bml,dSz_bml,& + syprt(ipt)%estr%hindex,syprt(ipt)%spindex,syprt(ipt)%estr%rho,syprt(ipt)%net_charge,syprt(ipt)%estr%coul_pot_r,& + syprt(ipt)%estr%coul_pot_k,tb%hubbardu,syprt(ipt)%estr%FSCOUL,lt%threshold) + + do i=1,gpat%sgraph(ipt)%llsize + GFPUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FPUL(:,i) + GFSCOUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FSCOUL(:,i) + SKForce(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%SKForce(:,i) + enddo + + call bml_deallocate(dSx_bml) + call bml_deallocate(dSy_bml) + call bml_deallocate(dSz_bml) + + call bml_deallocate(dH0x_bml) + call bml_deallocate(dH0y_bml) + call bml_deallocate(dH0z_bml) + + !if(.not.kernel%xlbolevel1)then + call bml_deallocate(syprt(ipt)%estr%rho) + call bml_deallocate(syprt(ipt)%estr%ham) + call bml_deallocate(syprt(ipt)%estr%ham0) + !endif + !call bml_deallocate(syprt(ipt)%estr%over) + !call bml_deallocate(syprt(ipt)%estr%zmat) + + enddo + + collectedforce = GFPUL + GFSCOUL + SKForce + + call gpmdcov_msMem("gpmdcov","Before steered MD (SMD) check",lt%verbose,myRank) + + !> Steered MD Force (is using SMD) + if(gpmdt%usesmd) then + + !> Allocate SMD Arrays + !! R1 and R2 for xyz coordinates for steered atoms in each pair + !! directional_smd_force for xyz force components + if(.not.allocated(R1))allocate(R1(3)) + if(.not.allocated(R2))allocate(R2(3)) + + do i = 1,gpmdt%smdnumpairs + !> Determine coordinates for steered atoms in each pair + R1 = sy%coordinate(:,gpmdt%smdatomind1(i)) + R2 = sy%coordinate(:,gpmdt%smdatomind2(i)) + + write(*,*) "gpmdcov_EnergAndForces SMD Pair Number ",i," Atoms ",gpmdt%smdatomind1(i),& + &" and ",gpmdt%smdatomind2(i) + !> Call constraints subroutine, harmonic to linear + !! collectedforce will be updated for steered atoms + call gpmdcov_constraint_harmonicToLinear(R1, R2, smd_total_force, smd_total_energy, lt%verbose) + + !> Update collectedforce to include SMD force for steered atoms + collectedforce(:,gpmdt%smdatomind1(i)) = collectedforce(:,gpmdt%smdatomind1(i)) + smd_total_force(:) + collectedforce(:,gpmdt%smdatomind2(i)) = collectedforce(:,gpmdt%smdatomind2(i)) - smd_total_force(:) + + !> Print velocity logging for steered atoms and total SMD energy/force terms + if(MDstep .gt. 2) then + call gpmdcov_msI("gpmdcov_EnergAndForces","Velocity of first steered atom " & + &//to_string(norm2(sy%velocity(:,gpmdt%smdatomind1(i)))), lt%verbose, myRank) + call gpmdcov_msI("gpmdcov_EnergAndForces","Velocity of second steered atom " & + &//to_string(norm2(sy%velocity(:,gpmdt%smdatomind2(i)))), lt%verbose, myRank) + endif + call gpmdcov_msI("gpmdcov_EnergAndForces","SMD Force Magnitude " & + & // to_string(norm2(smd_total_force)),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_EnergAndForces","SMD Total Energy " & + & // to_string(smd_total_energy),lt%verbose,myRank) + do k = 1,3 + dcoords(k) = modulo((R1(k) - R2(k)) + 0.5_dp*sy%lattice_vector(k,k),sy%lattice_vector(k,k)) - 0.5_dp * sy%lattice_vector(k,k) + enddo + dist = norm2(dcoords) + call gpmdcov_msI("gpmdcov_EnergAndForces","SMD distance " & + & // to_string(dist),lt%verbose,myRank) + + !> Add energy for current steered atom pair to total steering energy + !! This will then be added to the total energy of the system + smd_total_energy_allpairs = smd_total_energy_allpairs + smd_total_energy + enddo + + !> SMD Finite Difference Test + !! Only test for last SMD atom pair + !! Set testsmd to true to run finite difference derivative tests + testsmd = .false. + if(testsmd) then + write(*,*) "gpmdcov_EnergAndForces testing SMD" + call gpmdcov_msI("gpmdcov_EnergAndForces","Testing SMD Derivatives: SMD force (x-direction) " & + &//to_string(smd_total_force(1)), lt%verbose, myRank) + !! Test with three different delta h values + !! Current testing is only done in the x-direction by updating the x coordinate of R1 + deltas = (/0.001, 0.01, 0.1 /) + do k=1,3 + delta_h = deltas(k) + R1(1) = R1(1) + delta_h + call gpmdcov_constraint_harmonicToLinear(R1, R2, smd_test_force, smd_test_energy, & + lt%verbose) + energy_plus = smd_test_energy + R1(1) = R1(1) - 2.0_dp * delta_h + call gpmdcov_constraint_harmonicToLinear(R1, R2, smd_test_force, smd_test_energy, & + lt%verbose) + energy_minus = smd_test_energy + + !! Compute finite difference derivative of energy to compare to output force + denergy = (energy_minus - energy_plus)/(2.0_dp*delta_h) + call gpmdcov_msI("gpmdcov_EnergAndForces","Testing SMD Derivatives: d energy (x-direction) " & + &//to_string(denergy), lt%verbose, myRank) + differ = (smd_total_force(1) - denergy) + call gpmdcov_msI("gpmdcov_EnergAndForces","Testing SMD Derivatives: difference (force - denergy) " & + &//to_string(differ), lt%verbose, myRank) + call gpmdcov_msI("gpmdcov_EnergAndForces","Testing SMD Derivatives: delta h used " & + &//to_string(delta_h), lt%verbose, myRank) + + !> Reset R1(1) for next test + R1(1) = R1(1) + delta_h + enddo + endif + + !> Deallocate SMD Arrays + deallocate(R1) + deallocate(R2) + endif + + + mls_i = mls() + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + ! call prg_sumRealReduceN(collectedforce(1,:), sy%nats) + ! call prg_sumRealReduceN(collectedforce(2,:), sy%nats) + ! call prg_sumRealReduceN(collectedforce(3,:), sy%nats) + call prg_sumRealReduceN(collectedforce, sy%nats*3) + call prg_sumRealReduceN(ebandvector, gpat%TotalParts) + endif +#endif + + call gpmdcov_msI("gpmdcov_EnergAndForces","MPI rank finished prg_sumRealReduceN & + &for Forces"//to_string(mls() - mls_i),lt%verbose,myRank) + + coul_forces = coul_forces_r + coul_forces_k + + !> Get Repulsive energy and forces + ! call get_PairPot_contrib(sy%coordinate,sy%lattice_vector,sy%spindex,ppot,PairForces,ERep) + call get_PairPot_contrib_int(sy%coordinate,sy%lattice_vector,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,sy%spindex,ppot,PairForces,ERep) + + !> Get Coulombic energy + ECoul = 0.0; + do i = 1,sy%nats + ECoul = ECoul + charges(i)*(tb%hubbardu(sy%spindex(i))*charges(i) + coul_pot_r(i) + coul_pot_k(i) ); + enddo + + Etot = sum(ebandvector(:)) - 0.5_dp*ECoul + ERep + smd_total_energy_allpairs + + entropy = 0.0_dp + if(lt%Entropy)then + if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then + call gpmdcov_getentropy(evalsAll, dvalsAll, beta, Ef, myRank,entropy, lt%verbose) + else + write(*,*)"ERROR: Entropy calculation is only valid for MuCalcType= FromParts, or MuCalcType= Combined." + stop + endif + endif + + EPOT = Etot + entropy + + if(myRank == 1 .and. lt%verbose >= 2)then + write(*,*)"Energy Coulomb = ", ECoul + write(*,*)"Energy Band =", sum(ebandvector(:)) + write(*,*)"Energy Repulsive = ", ERep + write(*,*)"Energy Entropic = ", entropy + write(*,*)"Energy Electronic (Total) =", EPot + endif + + if(.not.allocated(sy%force))allocate(sy%force(3,sy%nats)) + + sy%force = collectedforce + PairForces + coul_forces + + if(myRank == 1 .and. lt%verbose >= 3)then + write(*,*)""; write(*,*)"FPUL + FSCOUL + SKForce" + do i = 1,sy%nats + write(*,*)"Collected Force",i,collectedforce(1,i),collectedforce(2,i),collectedforce(3,i) + enddo + + write(*,*)""; write(*,*)"Coulomb forces:" + do i = 1,sy%nats + write(*,*)"Coul Force",i,coul_forces(1,i),coul_forces(2,i),coul_forces(3,i) + enddo + + write(*,*)""; write(*,*)"Repulsive forces:" + do i = 1,sy%nats + write(*,*)i,PairForces(1,i),PairForces(2,i),PairForces(3,i) + enddo + + write(*,*)""; write(*,*) "Total forces" + do i = 1,sy%nats + write(*,*)i,sy%force(1,i),sy%force(2,i),sy%force(3,i) + enddo + endif + + deallocate(ebandvector) + + call gpmdcov_msMem("gpmdcov","After gpmd_EnergAndForces",lt%verbose,myRank) + + end subroutine gpmdcov_EnergAndForces + + subroutine gpmdcov_getentropy(evals, dvals, beta, mu, rank,entropy, verbose) + use gpmdcov_writeout_mod + implicit none + integer, parameter :: dp = kind(1.0d0) + integer :: i, norbs + integer, intent(in) :: rank + real(dp) :: fermi + real(dp), intent(in) :: evals(:), dvals(:) + real(dp), intent(in) :: beta + real(dp), intent(inout) :: mu, entropy + integer, optional, intent(in) :: verbose + real(dp), allocatable :: fvals(:) + + if (.not.allocated(fvals))then + allocate(fvals(size(evals))) + endif + + call gpmdcov_msMem("gpmdcov_getentropy","Getting entropic energy contribution ...",verbose,rank) + + norbs = size(evals, dim = 1) + + entropy = 0.0_dp + fermi = 0.0_dp + call gpmdcov_fermifunction(beta,evals,mu,fvals) + do i = 1,norbs + fermi = fvals(i) + if(abs(fermi) > 10d-9 .and. abs(fermi-1.0_dp) > 10d-9)then + entropy = entropy + (2.0_dp/beta)*dvals(i)*(fermi*log(fermi) + (1.0_dp-fermi)*log(1.0_dp-fermi)) + endif + enddo + + end subroutine gpmdcov_getentropy + +end module gpmdcov_EnergAndForces_mod + diff --git a/examples/gpmdk/src/gpmdcov_field.F90 b/examples/gpmdk/src/gpmdcov_field.F90 new file mode 100644 index 00000000..c00250f2 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_field.F90 @@ -0,0 +1,48 @@ +module gpmdcov_field + + public :: gpmdcov_apply_field + + contains + + !> Apply a constant field perturbation by changing the Coulombic potentials. + !! Units can be transformed by using the parameter \f$ \lambda \f$. + !! \param field Direction of the applied field (\f$ \hat{\textbf{E}} \f$). + !! \param intensity Intensity of the field (\f$ ||\textbf{E}|| \f$).. + !! \param coordinate Coordinates of the system (\f$ \textbf{r}\f$). + !! \param lambda Constant to premultiply the perturbation (\f$ \lambda\f$). + !! \param threshold Threshold value for bml format matrices. + !! \param verbose Different levels of verbosity. + !! + subroutine gpmdcov_apply_field(field,intensity,coordinates,lambda,final_coul_pot_r,verbose) + use gpmdcov_vars + + integer :: cont, l, nats + integer, intent(in) :: verbose + real(dp) :: fieldnorm, fieldx, fieldy, fieldz + real(dp) :: intensity, lambda, threshold + real(dp), intent(in) :: coordinates(:,:), field(3) + real(dp), allocatable, intent(inout) :: final_coul_pot_r(:) + + + write(*,*)'Applying a constant field perturbation ...' + + fieldx = field(1) + fieldy = field(2) + fieldz = field(3) + + fieldnorm=sqrt(fieldx**2+fieldy**2+fieldz**2) !Get the norm + + fieldx=intensity*fieldx/fieldnorm !Normalize and add intensity + fieldy=intensity*fieldy/fieldnorm + fieldz=intensity*fieldz/fieldnorm + + if(.not. allocated(final_coul_pot_r)) allocate(final_coul_pot_r(sy%nats)) + + do l=1,nats + final_coul_pot_r(l) = final_coul_pot_r(l) + lambda*(fieldx*coordinates(1,l) + & + & fieldy*coordinates(2,l) + fieldz*coordinates(3,l)) + enddo + + end subroutine gpmdcov_apply_field + +end module gpmdcov_field diff --git a/examples/gpmdk/src/gpmdcov_finalize.F90 b/examples/gpmdk/src/gpmdcov_finalize.F90 new file mode 100644 index 00000000..a87a39f7 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_finalize.F90 @@ -0,0 +1,195 @@ + !> Finalize gpmd + subroutine gpmdcov_Finalize() + + use gpmdcov_vars + + implicit none + integer :: io_max, io, m, isiostat + integer :: k, l + logical :: isopened + + if(bml_allocated(ham_bml))call bml_deallocate(ham_bml) + if(bml_allocated(ham0_bml))call bml_deallocate(ham0_bml) + if(bml_allocated(g_bml))call bml_deallocate(g_bml) + if(bml_allocated(rho_bml))call bml_deallocate(rho_bml) + if(bml_allocated(over_bml))call bml_deallocate(over_bml) + if(bml_allocated(zmat_bml))call bml_deallocate(zmat_bml) + + if(allocated(ppot))then + do k=1,sy%nsp + do l=1,sy%nsp + deallocate(ppot(k,l)%potparams) + enddo + enddo + deallocate(ppot) + endif + + if(allocated(gpat%sgraph))call prg_destroyGraphPartitioning(gpat) + if(allocated(sy%coordinate))call prg_destroy_system(sy) + if(allocated(sy%symbol)) deallocate(sy%symbol) + if(allocated(sy%atomic_number)) deallocate(sy%atomic_number) + if(allocated(sy%coordinate)) deallocate(sy%coordinate) + !if(allocated(sy%velocity)) deallocate(sy%velocity) + !if(allocated(sy%force)) deallocate(sy%force) + if(allocated(sy%net_charge)) deallocate(sy%net_charge) + if(allocated(sy%mass)) deallocate(sy%mass) + if(allocated(sy%lattice_vector)) deallocate(sy%lattice_vector) + !if(allocated(sy%recip_vector)) deallocate(sy%recip_vector) + if(allocated(sy%spindex)) deallocate(sy%spindex) + if(allocated(sy%splist)) deallocate(sy%splist) + if(allocated(sy%spatnum)) deallocate(sy%spatnum) + if(allocated(sy%spmass)) deallocate(sy%spmass) + !if(allocated(sy%userdef)) deallocate(sy%userdef) + !if(allocated(sy%resindex)) deallocate(sy%resindex) + !if(allocated(sy%resname)) deallocate(sy%resname) + !if(allocated(sy%atomname)) deallocate(sy%atomname) + + if(allocated(nl%nntype))call gpmdcov_destroy_nlist(nl,lt%verbose) + + if(allocated(syprt))then + do ipt=1,gpat%TotalParts + call prg_destroy_estr(syprt(ipt)%estr) + enddo + do ipt=1,gpat%TotalParts + call prg_destroy_subsystems(syprt(ipt),lt%verbose) + enddo + deallocate(syprt) + endif + + !call prg_progress_shutdown() + + if(allocated(TypeA))deallocate(TypeA) + if(allocated(TypeB))deallocate(TypeB) + if(allocated(intKind))deallocate(intKind) + if(allocated(hindex))deallocate(hindex) + if(allocated(hnode))deallocate(hnode) + if(allocated( TypeA ))deallocate( TypeA ) + if(allocated( TypeB ))deallocate( TypeB ) + if(allocated( intKind ))deallocate( intKind ) + if(allocated( CH_count))deallocate( CH_count ) + if(allocated( CH_count_cov))deallocate( CH_count_cov ) + if(allocated( Halo_count ))deallocate( Halo_count ) + if(allocated( Halo_count_cov ))deallocate( Halo_count_cov ) + if(allocated( PartsInRankI ))deallocate( PartsInRankI ) + if(allocated( adjncy ))deallocate( adjncy ) + if(allocated( adjncy_cov ))deallocate( adjncy_cov ) + if(allocated( core_count ))deallocate( core_count ) + if(allocated( core_count_cov ))deallocate( core_count_cov) + if(allocated( displ ))deallocate( displ) + if(allocated( hindex ))deallocate( hindex) + if(allocated( hnode ))deallocate( hnode ) + if(allocated( norbsInEachCH ))deallocate( norbsInEachCH ) + if(allocated( norbsInEachCHAtRank ))deallocate( norbsInEachCHAtRank ) + if(allocated( npartsVect ))deallocate( npartsVect ) + if(allocated( norbsInEachRank ))deallocate( norbsInEachRank ) + if(allocated( part ))deallocate( part ) + if(allocated( part_cov ))deallocate( part_cov ) + if(allocated( partsInEachRank ))deallocate( partsInEachRank ) + if(allocated( partsSizes))deallocate( partsSizes) + if(allocated( reshuffle ))deallocate( reshuffle ) + if(allocated( thisPartSize))deallocate( thisPartSize ) + if(allocated( vectorint ))deallocate( vectorint ) + if(allocated( xadj ))deallocate( xadj ) + if(allocated( xadj_cov ))deallocate( xadj_cov ) + if(allocated( GFPUL ))deallocate( GFPUL ) + if(allocated( GFSCOUL ))deallocate( GFSCOUL ) + if(allocated( Ker ))deallocate( Ker ) + if(allocated( PairForces ))deallocate( PairForces ) + if(allocated( SKForce ))deallocate( SKForce ) + if(allocated( VX ))deallocate( VX) + if(allocated( VY ))deallocate( VY) + if(allocated( VZ ))deallocate( VZ) + if(allocated( acceprat ))deallocate( acceprat ) + if(allocated( auxcharge ))deallocate( auxcharge ) + if(allocated( auxcharge1 ))deallocate( auxcharge1 ) + if(allocated( charges_old ))deallocate( charges_old ) + if(allocated( collectedforce ))deallocate( collectedforce ) + if(allocated( coul_forces ))deallocate( coul_forces ) + if(allocated( coul_forces_k ))deallocate( coul_forces_k ) + if(allocated( coul_forces_r ))deallocate( coul_forces_r ) + if(allocated( coul_pot_k ))deallocate( coul_pot_k ) + if(allocated( coul_pot_r ))deallocate( coul_pot_r ) + if(allocated( dqin ))deallocate( dqin ) + if(allocated( dqout ))deallocate( dqout ) + if(allocated( dvals ))deallocate( dvals ) + if(allocated( dvalsAll ))deallocate( dvalsAll ) + if(allocated( dvalsInRank ))deallocate( dvalsInRank ) + if(allocated( eigenvals ))deallocate( eigenvals ) + if(allocated( eigenvalues ))deallocate( eigenvalues ) + if(allocated( evals ))deallocate( evals ) + if(allocated( evalsAll ))deallocate( evalsAll ) + if(allocated( evalsInRank ))deallocate( evalsInRank ) + if(allocated( fvals ))deallocate( fvals ) + if(allocated( fvalsAll ))deallocate( fvalsAll ) + if(allocated( fvalsInRank ))deallocate( fvalsInRank ) + if(allocated( g_dense ))deallocate( g_dense ) + if(allocated( gbnd ))deallocate( gbnd ) + if(allocated( n ))deallocate( n ) + if(allocated( n_0 ))deallocate( n_0 ) + if(allocated( n_1 ))deallocate( n_1 ) + if(allocated( n_2 ))deallocate( n_2 ) + if(allocated( n_2 ))deallocate( n_2 ) + if(allocated( n_3 ))deallocate( n_3 ) + if(allocated( n_4 ))deallocate( n_4 ) + if(allocated( n_5 ))deallocate( n_5 ) + if(allocated( onsitesH ))deallocate( onsitesH ) + if(allocated( onsitesS ))deallocate( onsitesS ) + if(allocated( origin ))deallocate( origin ) + if(allocated( rhoat ))deallocate( rhoat ) + if(allocated( row ))deallocate( row ) + if(allocated( row1 ))deallocate( row1 ) + if(allocated( smdForce ))deallocate( smdForce ) + if(allocated(tb%splist))deallocate(tb%splist) + if(allocated(tb%numel))deallocate(tb%numel) + if(allocated(tb%basis))deallocate(tb%basis) + if(allocated(tb%onsite_energ))deallocate(tb%onsite_energ) + if(allocated(tb%mass))deallocate(tb%mass) + if(allocated(tb%HubbardU))deallocate(tb%HubbardU) + if(allocated(tb%w))deallocate(tb%w) + if(allocated(tb%norbi))deallocate(tb%norbi) + if(allocated(intPairsH))deallocate(intPairsH) + if(allocated(intPairsS))deallocate(intPairsS) + + if(bml_allocated( aux_bml ))call bml_deallocate( aux_bml ) + if(bml_allocated( dH0x_bml ))call bml_deallocate( dH0x_bml ) + if(bml_allocated( dH0y_bml ))call bml_deallocate( dH0y_bml ) + if(bml_allocated( dH0z_bml ))call bml_deallocate( dH0z_bml ) + if(bml_allocated( dSx_bml ))call bml_deallocate( dSx_bml ) + if(bml_allocated( dSy_bml ))call bml_deallocate( dSy_bml ) + if(bml_allocated( dSz_bml ))call bml_deallocate( dSz_bml ) + if(bml_allocated( g_bml ))call bml_deallocate( g_bml ) + if(bml_allocated( ham0_bml ))call bml_deallocate( ham0_bml ) + if(bml_allocated( ham_bml ))call bml_deallocate( ham_bml ) + if(bml_allocated( over_bml ))call bml_deallocate( over_bml ) + if(bml_allocated( rho_bml ))call bml_deallocate( rho_bml ) + if(bml_allocated( rhoat_bml))call bml_deallocate( rhoat_bml ) + if(bml_allocated( rhoh_bml ))call bml_deallocate( rhoh_bml ) + if(bml_allocated( zmat_bml ))call bml_deallocate( zmat_bml ) + if(bml_allocated( gch_bml ))call bml_deallocate( gch_bml ) + if(bml_allocated( copy_g_bml))call bml_deallocate( copy_g_bml ) + if(bml_allocated( gcov_bml ))call bml_deallocate( gcov_bml ) + if(bml_allocated( aux1_bml ))call bml_deallocate( aux1_bml ) + if(bml_allocated( ZK1_bml ))call bml_deallocate( ZK1_bml ) + if(bml_allocated( ZK2_bml ))call bml_deallocate( ZK2_bml ) + if(bml_allocated( ZK3_bml ))call bml_deallocate( ZK3_bml ) + if(bml_allocated( ZK4_bml ))call bml_deallocate( ZK4_bml ) + if(bml_allocated( ZK5_bml ))call bml_deallocate( ZK5_bml ) + if(bml_allocated( ZK6_bml ))call bml_deallocate( ZK6_bml ) + + + io_max = 100 + do io = io_max,1,-1 + inquire (unit=io, opened=isopened, iostat=isiostat) + if(isopened)then + if((io .ne. 5) .and. (io .ne. 6))then + write(*,*)"io",lib_init,io,isopened + !call flush(io) + !close(io) + endif + endif + end do + + + + end subroutine gpmdcov_Finalize + diff --git a/examples/gpmdk/src/gpmdcov_firstcharges.F90 b/examples/gpmdk/src/gpmdcov_firstcharges.F90 new file mode 100644 index 00000000..b79e84a0 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_firstcharges.F90 @@ -0,0 +1,131 @@ +!> First computation of charges. +!! \brief Here we compute the first "non-scf charges" based on H0. +!! +subroutine gpmdcov_FirstCharges(myeig) + + use gpmdcov_vars + use gpmdcov_mod + use gpmdcov_rhosolver_mod + use gpmdcov_writeout_mod + + logical :: myeig + + call gpmdcov_msMem("gpmdcov_FirstCharges","Before gpmd_FirstCharges",lt%verbose,myRank) + + if(.not.allocated(sy%net_charge))allocate(sy%net_charge(sy%nats)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!! + !Loop over parts +!!!!!!!!!!!!!!!!!!!!!!!!!!! + + sy%net_charge = 0.0_dp + + if(myeig)then + allocate(norbsInEachCHAtRank(partsInEachRank(myRank))) + norbsInEachCHAtRank = 0 + endif + +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + + norb = syprt(ipt)%estr%norbs + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%rho) + + !> Initialize the orthogonal versions of ham and rho. + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%oham) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%orho) + + + if(myeig)then + !> Orthogonalize ham. + if(lt%verbose >= 1 .and. myRank == 1) call prg_timer_start(ortho_timer) + call prg_orthogonalize(syprt(ipt)%estr%ham0,syprt(ipt)%estr%zmat,syprt(ipt)%estr%oham,& + lt%threshold,lt%bml_type,0) + if(lt%verbose >= 1 .and. myRank == 1) call prg_timer_stop(ortho_timer) + call gpmdcov_RhoSolver(syprt(ipt)%estr%oham,syprt(ipt)%estr%orho,syprt(ipt)%estr%evects) + norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) + else + call prg_build_density_fromEvalsAndEvects(syprt(ipt)%estr%evects, syprt(ipt)%estr%evals,& + & syprt(ipt)%estr%orho, lt%threshold, bndfil, lt%kbt, Ef, lt%verbose) + endif + + + !> Deorthogonalize rho. + if(lt%verbose >= 1 .and. myRank == 1) call prg_timer_start(deortho_timer) + call prg_deorthogonalize(syprt(ipt)%estr%orho,syprt(ipt)%estr%zmat,syprt(ipt)%estr%rho,& + lt%threshold,lt%bml_type,0) + if(lt%verbose >= 1 .and. myRank == 1) call prg_timer_stop(deortho_timer) + + !> Get charges based on rho. rho_bml is the input and sy%net_charge is the outputs vector containing + !! the charges. + call prg_get_charges(syprt(ipt)%estr%rho, syprt(ipt)%estr%over, syprt(ipt)%estr%hindex, syprt(ipt)%net_charge, tb%numel,& + syprt(ipt)%spindex, lt%mdim, lt%threshold) + + if(lt%verbose.GE.4 .and. myRank == 1)then + write(*,*)"" + write(*,*)"Total charge of part "//to_string(ipt) & + //" = "//to_string(sum(syprt(ipt)%net_charge(:))) + write(*,*)"" + write(*,*)""; write(*,*)"Part charges:" + do j=1,syprt(ipt)%nats + write(*,*)syprt(ipt)%symbol(j),syprt(ipt)%net_charge(j) + enddo + endif + + do j=1,gpat%sgraph(ipt)%llsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + sy%net_charge(jj) = syprt(ipt)%net_charge(j) + enddo + + if(myeig) call bml_deallocate(syprt(ipt)%estr%oham) + call bml_deallocate(syprt(ipt)%estr%orho) + call bml_deallocate(syprt(ipt)%estr%rho) + + enddo + + ! End of loop over parts. + mls_i = mls() + + ! Get the chemical potential. Feb 2021 implemetation + if(eig .and. lt%MuCalcType == "FromParts") call gpmdcov_muFromParts() + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(sy%net_charge, sy%nats) + endif +#endif + + call gpmdcov_msIII("gpmdcov_FirstCharges","MPI rank finished prg_sumIntReduceN for qs ="& + &//to_string(mls() - mls_i),lt%verbose,myRank) + + !> Gather charges from all the parts. + if(.not.allocated(charges_old))allocate(charges_old(sy%nats)) + + if (printRank() == 1) then + if(lt%verbose >= 2)then + write(*,*)"" + write(*,*)"Total charge of the system = "//to_string(sum(sy%net_charge(:))) + write(*,*)"" + if(lt%verbose >= 5) call prg_write_system(sy,"charged_system","pdb") + write(*,*)""; write(*,*)"Full System charges:" + do j=1,sy%nats + write(*,*)j,sy%symbol(j),sy%net_charge(j) + enddo + endif + endif + + sy%net_charge(:) = sy%net_charge(:)- sum(sy%net_charge(:))/real(sy%nats) + + charges_old = sy%net_charge + call gpmdcov_msMem("gpmdcov", "After gpmd_FirstCharges",lt%verbose,myRank) + +end subroutine gpmdcov_FirstCharges + + diff --git a/examples/gpmdk/src/gpmdcov_graphpart.F90 b/examples/gpmdk/src/gpmdcov_graphpart.F90 new file mode 100644 index 00000000..66e54269 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_graphpart.F90 @@ -0,0 +1,140 @@ + + !> Graph partitioning subroutine + subroutine gpmd_graphpart + use gpmdcov_vars + use gpmdcov_writeout_mod + + integer :: tnnz + + !> Create graph partitioning - Use Block or METIS or METIS+SA or METIS+KL + if(lt%verbose >= 1) call prg_timer_start(part_timer) + + !> Block partitioning + if (gsp2%partition_type == "Block") then + !> Partition by orbital or atom + if (gsp2%graph_element == "Atom") then + if(gsp2%partition_count == 1)then + gpat%TotalParts = 1 + gsp2%nodesPerPart = sy%nats + endif + call prg_equalPartition(gpat, gsp2%nodesPerPart, sy%nats) + else + call prg_equalGroupPartition(gpat, hindex, nnodes, gsp2%nodesPerPart, sy%nats) + endif + + !> METIS, METIS+SA, or METIS+KL partitioning + else + +#ifdef DO_GRAPHLIB + + if(.not.allocated(xadj))then + + allocate(xadj(sy%nats+1)) + + tnnz = 0 !This must be done at the bml level + do i=1,sy%nats + tnnz = tnnz + bml_get_row_bandwidth(g_bml,i) + enddo + + allocate(adjncy(tnnz+1)) + + call bml_adjacency(g_bml, xadj, adjncy, 1) + + nnodes = sy%nats + else + nnodes = sy%nats + endif + + nparts = gsp2%partition_count + + if (first_part) then + allocate(part(nnodes)) + allocate(core_count(nparts)) + endif + + if(allocated(CH_count)) deallocate(CH_count) + if(allocated(Halo_count)) deallocate(Halo_count) + allocate(CH_count(nparts)) + allocate(Halo_count(nparts, nnodes)) + + !> For METIS, if no refinement of first time, do full partitioning + if (gsp2%partition_refinement == 'None' .or. first_part .or. mdstep <= 1) then + call gpmdcov_message("graph_part","Will do METIS partition",lt%verbose,2,myrank) + + call prg_destroyGraphPartitioning(gpat) + + !> Which METIS partitioning + select case(gsp2%partition_type) + + case("Block") + if (gsp2%graph_element == "Atom") then + call prg_equalPartition(gpat, gsp2%nodesPerPart, sy%nats) + else + call prg_equalGroupPartition(gpat, hindex, nnodes, gsp2%nodesPerPart, sy%nats) + endif + + case("METIS") + call prg_metisPartition(gpat, sy%nats, sy%nats, xadj, adjncy, nparts, part, core_count,& + CH_count, Halo_count, sumCubes, maxCH, smooth_maxCH, pnorm) + + case("METIS+SA") + + call prg_metisPartition(gpat, sy%nats, sy%nats, xadj, adjncy, nparts, part, core_count,& + CH_count, Halo_count, sumCubes, maxCH, smooth_maxCH, pnorm) + + seed = mdstep + call prg_simAnnealing(gpat, xadj, adjncy, part, core_count, CH_count, & + Halo_count, sumCubes, maxCH,smooth_maxCH,pnorm,1,seed) + + case("METIS+KL") + call prg_metisPartition(gpat, sy%nats, sy%nats, xadj, adjncy, nparts, part, core_count,& + CH_count, Halo_count, sumCubes, maxCH, smooth_maxCH, pnorm) + call prg_KernLin2(gpat, xadj, adjncy, part, core_count, CH_count, & + Halo_count, sumCubes, maxCH, smooth_maxCH, pnorm) + + case default + write(*,*)"No METIS partitioning specified" + stop "" + end select + + first_part = .false. + + !> Not first time, do refinement + else + call gpmdcov_message("graph_part","In refinment",lt%verbose,1,myrank) + if (.not. first_part) then + + !> Which refinement + select case(gsp2%partition_refinement) + case("SA") + call prg_simAnnealing(gpat, xadj, adjncy, part, core_count, CH_count, & + Halo_count, sumCubes, maxCH,smooth_maxCH,pnorm, 10, seed) + case("KL") + call prg_KernLin2(gpat, xadj, adjncy, part, core_count, CH_count, & + Halo_count, sumCubes, maxCH, smooth_maxCH, pnorm) + case default + write(*,*)"No refinement specified" + stop "" + end select + + endif + endif + + deallocate(xadj) + deallocate(adjncy) + deallocate(CH_count) + deallocate(Halo_count) +#else + write(*,*)"ERROR: METIS option will only work if the code is compiled with GRAPHLIB" +#endif + + endif + + !write(*,*)"PARTITION" + !write(*,*)gpat%sgraph(1)%nodeInPart + + + if(lt%verbose >= 1) call prg_timer_stop(part_timer) + + end subroutine gpmd_graphpart + diff --git a/examples/gpmdk/src/gpmdcov_highorder.F90 b/examples/gpmdk/src/gpmdcov_highorder.F90 new file mode 100644 index 00000000..535cc889 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_highorder.F90 @@ -0,0 +1,248 @@ +!> Compute higher order electronic properties, such as polarizability. +!! +module gpmdcov_highorder_mod + + ! Local modules + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_prepareMD_mod + use gpmdcov_mdloop_mod + use gpmdcov_init_mod + use gpmdcov_part_mod + use gpmdcov_assert_mod + use gpmdcov_mod + use gpmdcov_diagonalize_mod + use gpmdcov_kernel_mod + ! Progress modules + use prg_ptable_mod + use prg_response_mod + + + public :: gpmdcov_get_born_charges_v1 + public :: gpmdcov_get_born_charges_v2 + +contains + + !> Computes the Born charges + !! \brief Based on the derivative of the dipole moment with respect + !! to the coordinates. + !! \param dipoleMoment Dipole moment of the molecule. + !! \param bornCharges for each atom + subroutine gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) + + implicit none + real(dp),allocatable :: dipoleMoment(:),savedCharges(:),bornCharges(:,:) + real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) + real(dp) :: factor,dr,dfx,dfy,dfz + integer :: atomI + + if(.not. allocated(dipoleMoment)) allocate(dipoleMoment(3)) + allocate(savedDipoleMoment(3)) + factor = 1.0_dp + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + write(*,*)"Dipole Moment (e * Ang)=",dipoleMoment + savedDipoleMoment = dipoleMoment + + !Born Charges + allocate(savedCoords(3,sy%nats)) + allocate(savedCharges(sy%nats)) + if(.not. allocated(bornCharges)) allocate(bornCharges(9,sy%nats)) + savedCoords = sy%coordinate + savedCharges = sy%net_charge + + dr = 0.000001 + do atomI = 1,sy%nats + + !write(*,*)"BornCharges",atomI,"----------------------------------" + sy%coordinate(1,atomI) = sy%coordinate(1,atomI) + dr + call gpmdcov_InitParts() + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + + dfx = (dipoleMoment(1) - savedDipoleMoment(1))/dr + dfy = (dipoleMoment(2) - savedDipoleMoment(2))/dr + dfz = (dipoleMoment(3) - savedDipoleMoment(3))/dr + sy%coordinate(1,atomI) = savedCoords(1,atomI) + !write(*,*)"BornCharges",dfx,dfy,dfz,savedCharges(atomI) + bornCharges(1,atomI) = dfx + bornCharges(2,atomI) = dfy + bornCharges(3,atomI) = dfz + + sy%coordinate(2,atomI) = sy%coordinate(2,atomI) + dr + call gpmdcov_InitParts() + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + dfx = (dipoleMoment(1) - savedDipoleMoment(1))/dr + dfy = (dipoleMoment(2) - savedDipoleMoment(2))/dr + dfz = (dipoleMoment(3) - savedDipoleMoment(3))/dr + sy%coordinate(2,atomI) = savedCoords(2,atomI) + !write(*,*)"BornCharges",dfx,dfy,dfz,savedCharges(atomI) + bornCharges(4,atomI) = dfx + bornCharges(5,atomI) = dfy + bornCharges(6,atomI) = dfz + + sy%coordinate(3,atomI) = sy%coordinate(3,atomI) + dr + call gpmdcov_InitParts() + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + dfx = (dipoleMoment(1) - savedDipoleMoment(1))/dr + dfy = (dipoleMoment(2) - savedDipoleMoment(2))/dr + dfz = (dipoleMoment(3) - savedDipoleMoment(3))/dr + sy%coordinate(3,atomI) = savedCoords(3,atomI) + !write(*,*)"BornCharges",dfx,dfy,dfz,savedCharges(atomI) + bornCharges(7,atomI) = dfx + bornCharges(8,atomI) = dfy + bornCharges(9,atomI) = dfz + !write(*,*)"BornCharges-------------------------------------------" + + enddo + + deallocate(savedCoords) + deallocate(savedDipoleMoment) + + + end subroutine gpmdcov_get_born_charges_v1 + + !> Born charges + !! \brief Will compute the Born charges based on the derivatives + !! of the forces with respect to the field. + subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) + implicit none + real(dp),allocatable :: dipoleMoment(:),savedCharges(:),bornCharges(:,:) + real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) + real(dp) :: lambda,intensity,field(3),threshold + real(dp), allocatable :: polarizability(:), forcesSaved(:,:) + integer :: atomI, verbose + integer, allocatable, intent(in) :: norbi(:) + type(bml_matrix_t) :: prt_bml + real(dp) :: relperm, keconst, tfact + real(dp) :: intensity0,dintensity + + lambda = 1.0_dp !2.0_dp/14.3996437701414_dp + threshold = 0.0_dp + verbose = 3 + + if(.not. allocated(bornCharges)) allocate(bornCharges(9,sy%nats)) + + !This will assume only one part + call gpmdcov_InitParts() + allocate(forcesSaved(3,sy%nats)) + + intensity0 = 0.0000_dp !eV/Ang + dintensity = 0.001_dp + intensity = 1.0D-4 + bornCharges = 0.0_dp + + !We will compute dF/dE in x direction + field = 0.0_dp ; field(1) = -1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + !call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ! ,prt_bml,threshold,sy%spindex,norbi,verbose) + sy%net_charge = 0.0_dp + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + forcesSaved = sy%force + field = 0.0_dp ; field(1) = 1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + sy%net_charge = 0.0_dp + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + bornCharges(1,:) = (sy%force(1,:) - forcesSaved(1,:))/(2.0_dp*intensity) + bornCharges(2,:) = (sy%force(2,:) - forcesSaved(2,:))/(2.0_dp*intensity) + bornCharges(3,:) = (sy%force(3,:) - forcesSaved(3,:))/(2.0_dp*intensity) + + !We will compute dF/dE in y direction + field = 0.0_dp ; field(2) = -1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + forcesSaved = sy%force + field = 0.0_dp ; field(2) = 1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + bornCharges(4,:) = (sy%force(1,:) - forcesSaved(1,:))/(2.0_dp*intensity) + bornCharges(5,:) = (sy%force(2,:) - forcesSaved(2,:))/(2.0_dp*intensity) + bornCharges(6,:) = (sy%force(3,:) - forcesSaved(3,:))/(2.0_dp*intensity) + + !We will compute dF/dE in z direction + field = 0.0_dp ; field(3) = -1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + forcesSaved = sy%force + field = 0.0_dp ; field(3) = 1.0_dp + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + bornCharges(7,:) = (sy%force(1,:) - forcesSaved(1,:))/(2.0_dp*intensity) + bornCharges(8,:) = (sy%force(2,:) - forcesSaved(2,:))/(2.0_dp*intensity) + bornCharges(9,:) = (sy%force(3,:) - forcesSaved(3,:))/(2.0_dp*intensity) + + relperm = 1.0_dp; + !! The 14.399 factor corresponds to 1/(4*pi*epsilon0) in eV*Ang + keconst = 14.3996437701414_dp*relperm; + + deallocate(forcesSaved) + + end subroutine gpmdcov_get_born_charges_v2 + +end module gpmdcov_highorder_mod diff --git a/examples/gpmdk/src/gpmdcov_init.F90 b/examples/gpmdk/src/gpmdcov_init.F90 new file mode 100644 index 00000000..db0b4114 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_init.F90 @@ -0,0 +1,243 @@ +module gpmdcov_Init_mod + +contains + + !> Initialize the program variables and parse input files. + !! + subroutine gpmdcov_Init(lib_on) + + use gpmdcov_vars + use gpmdcov_writeout_mod + use gpmdcov_kernel_mod + use gpmdcov_neighbor_mod + use gpmdcov_parser_mod + use gpmdcov_allocation_mod + use gpmdcov_prepareMD_mod, only : gpmdcov_addVelocity + + integer :: ix, iy, iz, numreps + type(system_type) :: syaux + real(dp) :: minEdge + + logical, intent(in) :: lib_on + + !> Start progress + ! This will initialize the MPI ranks + if(.not. lib_init) call prg_progress_init() + + !> Get MPI rank +#ifdef DO_MPI + myRank = getMyRank() + 1 + numRanks = getNRanks() +#else + myRank = 1 + numRanks = 1 +#endif + + call gpmdcov_msI("gpmdcov_Init","GPMD started ...",lt%verbose,myRank) + + !> Get the input file from argumets. + if(lib_on)then + inputfile = "gpmd_input.in" + else + call gpmdcov_msI("gpmdcov_Init","Reading inputfile ...",lt%verbose,myRank) + call getarg(1,inputfile) + endif + + if(.not. lib_init)then + + !> Parsing input file. This file contains all the variables needed to + ! run the scf including the sp2 (solver) variables. lt is "latte_type" structure + ! containing all the variables. + call parse_latte(lt,inputfile) + + !> Parsing SP2 input paramenters. This will read the variables in the input file. + ! sp2 is the "sp2data_type". + call prg_parse_sp2(sp2,inputfile) + + !> Parsing GSP2 input paramenters. This will read the variables in the input file. + ! gsp2 is the "gsp2data_type". These parameters control all the graph + ! related variables incluind partition methos, number of parts, etc. + call prg_parse_gsp2(gsp2,inputfile) + + !> Parsing Extended Lagrangian input paramenters. This will read all the + ! variables related to the Extended lagrangian method. + ! xl is the "xlbo_type". + call prg_parse_xlbo(xl,inputfile) + + !> Parsing Z sparse propagation. This will read all the variables related to + ! the propagation and construction of the inverse overlap. + call prg_parse_zsp(zsp,inputfile) + + !> Parsing specific variales for the gpmd code + call gpmdcov_parse(trim(adjustl(inputfile)),gpmdt) + + !> Parse variables for the kernel method + call gpmdcov_parseKernel(kernel,inputfile) + endif + + + !> Parsing system coordinates. This reads the coords.pdb file to get the position of every + ! atom in the system. sy is the "system_type" structure containing all the variables. + + ! Reading the system + if(gpmdt%replicatex + gpmdt%replicatey + gpmdt%replicatez > 0)then + if(.not. lib_on) call prg_parse_system(syaux,lt%coordsfile) + call prg_replicate_system(syaux,sy,gpmdt%replicatex,gpmdt%replicatey,gpmdt%replicatez) + call gpmdcov_reallocate_intVect(sy%atomic_number,sy%nats) + call gpmdcov_reallocate_realVect(sy%mass,sy%nats) + call gpmdcov_reallocate_charVect(sy%splist,syaux%nsp) + call gpmdcov_reallocate_realVect(sy%spmass,syaux%nsp) + call gpmdcov_reallocate_intVect(sy%spatnum,syaux%nsp) + sy%nsp = syaux%nsp + sy%splist = syaux%splist + sy%spmass = syaux%spmass + sy%spatnum = syaux%spatnum + + if(gpmdt%htod)then + numh = 0 + do i = 1,sy%nats + if(syaux%atomic_number(i) .eq. 1)then + syaux%mass(i) = 2.013553212745 ! Change H mass to D mass + numh = numh + 1 + endif + enddo + call gpmdcov_msI("gpmdcov_init","Changing H mass to D mass for "& + &//to_string(numh)//" H atoms",lt%verbose,myRank) + endif + + numreps = 0 + do ix = 1,gpmdt%replicatex + do iy = 1,gpmdt%replicatey + do iz = 1,gpmdt%replicatez + numreps = numreps + 1 + sy%atomic_number(1 + (numreps - 1)*syaux%nats:numreps*syaux%nats) = syaux%atomic_number(:) + sy%mass(1 + (numreps - 1)*syaux%nats:numreps*syaux%nats) = syaux%mass(:) + enddo + enddo + enddo + + call prg_destroy_system(syaux) + if (myRank == 1) then + if(lt%verbose >= 2) call prg_write_system(sy,adjustl(trim(lt%jobname))//"_postReplication","pdb") + endif + + else + + if(.not. lib_on) call prg_parse_system(sy,lt%coordsfile) + + if(gpmdt%htod)then + numh = 0 + do i = 1,sy%nats + if(sy%atomic_number(i) .eq. 1)then + sy%mass(i) = 2.013553212745 ! Change H mass to D mass + numh = numh + 1 + endif + enddo + call gpmdcov_msI("gpmdcov_init","Changing H mass to D mass for "& + &//to_string(numh)//" H atoms",lt%verbose,myRank) + endif + endif + + call gpmdcov_msI("gpmdcov_init","Number of atoms = "//to_string(sy%nats),lt%verbose,myRank) + + + !> This variable sets the maximun number of non-zeros per row. I will be + ! used when sparse matrices are used. Typically sparse matrices will only be + ! used for storing the graph and neighbor list. + if(gsp2%mdim > 0)then + mdim = gsp2%mdim + else + mdim = sy%nats + endif + + if(.not.allocated(origin)) allocate(origin(3)) + !> Center sytem inside the box and fold it by the lattice_vectors. This is + ! done only for visualization purposes. + !origin = 0.0_dp + if(gpmdt%trfl)then + call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) + endif + !origin = 0.0_dp + + if(gpmdt%restartfromdump)then + call gpmdcov_msI("gpmdcov_init","Restarting from dump file ",lt%verbose,myRank) + call gpmdcov_restart() + endif + + !> This will construct a pdb file with the system that was read by the code. + ! It should be used to verify that the program indeed is reading the system + ! of interest. + if (myRank == 1) then + if(lt%verbose >= 2) call prg_write_system(sy,adjustl(trim(lt%jobname))//"_centered","pdb") + endif + + !> Get the Coulombic cut off. + call get_coulcut(lt%coul_acc,lt%timeratio,sy%nats,sy%lattice_vector,coulcut) + + !> Building the neighbor list. + call gpmdcov_msMem("gpmdcov_init","Before build_nlist",lt%verbose,myRank) + mls_i = mls() + + minEdge = min(norm2(sy%lattice_vector(1,:)),norm2(sy%lattice_vector(2,:)),norm2(sy%lattice_vector(3,:))) + if(coulcut < 2*minEdge) nlistSparse = .true. + nlistSparse = .false. + if(nlistSparse)then + call gpmdcov_msI("gpmdcov_init", "Doing Linear Scaling Neighbor list construction... ",lt%verbose,myRank) + call gpmdcov_build_nlist_sparse_v2(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) + else + call gpmdcov_msI("gpmdcov_init", "Doing Full Neighbor list construction... ",lt%verbose,myRank) + call gpmdcov_build_nlist_full(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) + endif + !call gpmdcov_build_nlist_sparse(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) + !call build_nlist_int(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose) + ! LBox(1) = sy%lattice_vector(1,1) + ! LBox(2) = sy%lattice_vector(2,2) + ! LBox(3) = sy%lattice_vector(3,3) + + ! call gpmd_nearestneighborlist(nl%nrnnlist,nl%nndist,nl%nnRx,nl%nnRy,nl%nnRz,nl%nnType, & + ! &sy%coordinate(1,:),sy%coordinate(2,:),sy%coordinate(3,:),LBox,coulcut,sy%nats,200) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_nlist "& + &//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_init","After build_nlist",lt%verbose,myRank) + + !> LATTE Hamiltonian parameter + call load_latteTBparams(tb,sy%splist,lt%parampath) + + !> Get the reciprocal vectors + call prg_get_recip_vects(sy%lattice_vector,sy%recip_vector,sy%volr,sy%volk) + + !> Bond integrals parameters for LATTE Hamiltonian. + call gpmdcov_msMem("gpmdcov","Before load_bintTBparamsH",lt%verbose,myRank) + call load_bintTBparamsH(sy%splist,tb%onsite_energ,& + typeA,typeB,intKind,onsitesH,onsitesS,intPairsH,intPairsS,lt%parampath) + call write_bintTBparamsH(typeA,typeB,& + intKind,intPairsH,intPairsS,adjustl(trim(lt%jobname))//"_mybondints.nonorth") + + !> Load Pair potentials for LATTE TB. + call load_PairPotTBparams(lt%parampath,sy%splist,ppot,lt%verbose) + !write(*,*)"ppot",ppot(1,1)%potparams(:) + !stop + + !> Allocate bounds vector. + allocate(gbnd(2)) + + !> mdstep needs to be initialized. + mdstep = 0 + + !> This is just to get the number of total orbitals + call get_hindex(sy%spindex,tb%norbi,sy%estr%hindex,norb,lt%verbose) + sy%estr%norbs = norb + call gpmdcov_msRel("Total Number of Orbitals:",real(sy%estr%norbs,dp),lt%verbose,myRank) + + !> Get total occupation + ! WARNING: This could change depending on the TB method being used. + + sy%estr%nel = sum(element_numel(sy%atomic_number(:))) + bndfilTotal = sy%estr%nel/(2.0_dp*norb) + call gpmdcov_msRel("Total Number of Electrons:",real(sy%estr%nel,dp),lt%verbose,myRank) + + call gpmdcov_msMem("gpmdcov","After gpmd_Init",lt%verbose,myRank) + + end subroutine gpmdcov_Init + +end module gpmdcov_Init_mod diff --git a/examples/gpmdk/src/gpmdcov_initparts.F90 b/examples/gpmdk/src/gpmdcov_initparts.F90 new file mode 100644 index 00000000..f0e71c32 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_initparts.F90 @@ -0,0 +1,116 @@ +!> Initialize the partition. +!! +subroutine gpmdcov_InitParts + use gpmdcov_vars + use gpmdcov_writeout_mod +#ifdef USE_LATTE + use gpmdcov_latte_mod +#endif + implicit none + integer :: norbsCore + + call gpmdcov_msMem("gpmdcov","Before gpmd_InitParts",lt%verbose,myRank) + + !Vectorizing part sizes + if(allocated(partsSizes)) deallocate(partsSizes) + allocate(partsSizes(gpat%TotalParts)) + + partsSizes=0.0_dp + do i = 1,gpat%TotalParts + partsSizes(i) = gpat%sgraph(i)%llsize + enddo + +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + + if(lt%verbose >= 3)then + write(*,*)"" + write(*,*)" #####################################" + write(*,*)" (rank "//to_string(myRank)//") Initializing partition "//to_string(ipt) + write(*,*)" #####################################" + write(*,*)"" + end if + + if(lt%verbose >= 3)then + write(*,*)"Number of atoms in the core = "//to_string(gpat%sgraph(ipt)%llsize) + write(*,*)"Number of atoms in the core+halo = "//to_string(gpat%sgraph(ipt)%lsize) + write(*,*)"" + end if + + !> Get the mapping of the Hamiltonian index with the atom index + if(allocated(syprt(ipt)%estr%hindex))deallocate(syprt(ipt)%estr%hindex) + allocate(syprt(ipt)%estr%hindex(2,syprt(ipt)%nats)) + + !call get_hindex(syprt(ipt)%spindex,tb%norbi,syprt(ipt)%estr%hindex,norb,norbCores) + + !We use a new routine to get the number of orbitals in the core + call get_hindex_coreHalo(syprt(ipt)%spindex,gpat%sgraph(ipt)%llsize,tb%norbi,syprt(ipt)%estr%hindex,norb,norbsCore,lt%verbose) + syprt(ipt)%estr%norbs = norb + syprt(ipt)%estr%norbsCore = norbsCore + + if(bml_allocated(syprt(ipt)%estr%ham0))then + call bml_deallocate(syprt(ipt)%estr%ham0) + call bml_deallocate(syprt(ipt)%estr%over) + endif + + call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%ham0) + call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%over) + + !Construction of the hamiltonian for every part +#ifdef USE_LATTE + call get_hsmat_latte(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%coordinate,syprt(ipt)%symbol,& + syprt(ipt)%lattice_vector,lt%threshold) +#else + if(gpmdt%usevectsk)then + call get_hsmat_vect(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%coordinate,& + syprt(ipt)%lattice_vector,syprt(ipt)%spindex,& + tb%norbi,syprt(ipt)%estr%hindex,onsitesH,onsitesS,intPairsH,intPairsS,lt%threshold) + else + call get_hsmat(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%coordinate,& + syprt(ipt)%lattice_vector,syprt(ipt)%spindex,& + tb%norbi,syprt(ipt)%estr%hindex,onsitesH,onsitesS,intPairsH,intPairsS,lt%threshold) + endif +#endif + + if (myRank == 1 .and. lt%verbose >= 5)then + write(*,*)"H0 and S for part:" + call bml_print_matrix("H0",syprt(ipt)%estr%ham0,0,6,0,6) + call bml_print_matrix("S",syprt(ipt)%estr%over,0,6,0,6) + if (lt%verbose >= 6)then + call bml_write_matrix(syprt(ipt)%estr%ham0,"H0.mtx") + call bml_write_matrix(syprt(ipt)%estr%over,"S.mtx") + ! stop + endif + endif + + !> Get occupation based on last shell population. + ! WARNING: This could change depending on the TB method being used. + !nel = sum(element_numel(syprt(ipt)%atomic_number(:)),& + ! size(syprt(ipt)%atomic_number,dim=1)) + nel = sum(element_numel(syprt(ipt)%atomic_number(:))) + bndfil = nel/(2.0_dp*norb) + + !> Initialize the density matrix (rho_bml) and inverse overlap factor (zmat_bml). + if(bml_allocated(syprt(ipt)%estr%zmat)) call bml_deallocate(syprt(ipt)%estr%zmat) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%zmat) + + !> Get the Inverse square root overlap matrix. + if(lt%verbose >= 3) call prg_timer_start(dyn_timer,"Build Z for part") + call gpmdcov_buildz(syprt(ipt)%estr%over,syprt(ipt)%estr%zmat) + if(lt%verbose >= 3) call prg_timer_stop(dyn_timer,1) + + if(myRank == 1 .and. lt%verbose >= 5)then + write(*,*)"Z matrix for part:" + call bml_print_matrix("Z",syprt(ipt)%estr%zmat,0,6,0,6) + endif + + enddo + call gpmdcov_msMem("gpmdcov","After gpmd_InitParts",lt%verbose,myRank) + +end subroutine gpmdcov_InitParts + diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 new file mode 100644 index 00000000..0ed8066b --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -0,0 +1,1272 @@ +!> Kernel Preconditioner related routined. +!! \brief This module will be used to compute quantities related to the +!! Kernel preconditioner. +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_kernel_mod + + use prg_kernelparser_mod + use gpmdcov_allocation_mod + use gpmdcov_writeout_mod + + implicit none + + private + + integer, parameter :: dp = kind(1.0d0) + + public :: gpmdcov_getKernel, gpmdcov_getKernel_byBlocks, gpmdcov_parseKernel + public :: gpmdcov_applyKernel, gpmdcov_getKernel_byParts, gpmdcov_rankN_update_byParts + + !> General latte input variables type. + !! + type, public :: kernel_type + + !> Kernel type (if full or other) + character(100) :: kernelType + + !> Kernel initial mixing method + character(100) :: initialMixingWith + + !> Verbosity level. + integer :: verbose + + !> Threshold values for matrix elements. + real(dp) :: threshold + + !> Use in mixing. + logical :: kernelMixing + + !> Update each + integer :: updateEach + + !> If the full preconditioner needs to be always built + logical :: buildAlways + + !> Number of rank N updates + integer :: rankNUpdate + + !> If Update after build + logical :: updateAfterBuild + + !> XLBO level. This adds an extra scf which increases stability + logical :: xlbolevel1 + + end type kernel_type + + type(kernel_type), public :: kernel + +contains + + + !> The parser for the kernel type. + !! + subroutine gpmdcov_parseKernel(kernel,filename) + + implicit none + type(kernel_type) :: kernel + integer, parameter :: nkey_char = 2, nkey_int = 3, nkey_re = 1, nkey_log = 4 + character(len=*) :: filename + + !Library of keywords with the respective defaults. + character(len=50), parameter :: keyvector_char(nkey_char) = [character(len=50) :: & + 'KernelType=','InitialMixingWith='] + character(len=100) :: valvector_char(nkey_char) = [character(len=100) :: & + 'Full','Lin'] + + character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & + 'Verbose=', 'UpdateEach=', 'RankNUpdate='] + integer :: valvector_int(nkey_int) = (/ & + 0 , 0, 0 /) + + character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & + 'Threshold='] + real(dp) :: valvector_re(nkey_re) = (/& + 0.00001 /) + + character(len=50), parameter :: keyvector_log(nkey_log) = [character(len=50) :: & + 'KernelMixing=','BuildAlways=','UpdateAfterBuild=',"XLBOLevel1="] + logical :: valvector_log(nkey_log) = (/& + .false., .false.,.false.,.false./) + + !Start and stop characters + character(len=50), parameter :: startstop(2) = [character(len=50) :: & + 'KERNEL{', '}'] + + call prg_parsing_kernel(keyvector_char,valvector_char& + ,keyvector_int,valvector_int,keyvector_re,valvector_re,& + keyvector_log,valvector_log,trim(filename),startstop) + + !Characters + kernel%kernelType = valvector_char(1) + kernel%initialMixingWith = valvector_char(2) + + !Integers + kernel%verbose = valvector_int(1) + kernel%updateEach = valvector_int(2) + kernel%rankNUpdate = valvector_int(3) + + !Reals + kernel%threshold = valvector_re(1) + + !Logical + kernel%kernelmixing = valvector_log(1) + kernel%buildAlways = valvector_log(2) + kernel%updateAfterBuild = valvector_log(3) + kernel%xlbolevel1 = valvector_log(4) + + end subroutine gpmdcov_parseKernel + + + !> Subroutine for computing the Kernel Preconditioner. + !! \brief This particular routine builds the full kernel + !! considering the system as a whole. For every perturbation + !! on atom "i" partially fills column "i" of a full Jacobian + !! (inverse of the Kernel). Paralelzation is obtained over + !! computing the perturbation for the part. + subroutine gpmdcov_getKernel(nats) + use gpmdcov_vars + use gpmdcov_writeout_mod + use prg_response_mod + real(dp), allocatable :: chargePertVect(:) + real(dp), allocatable :: my_coul_forces_k(:,:) + real(dp), allocatable :: my_coul_forces_r(:,:), my_coul_pot_k(:), my_coul_pot_r(:) + real(dp), allocatable :: Jacob(:), work(:), ptcoul_pot_k(:),ptcoul_pot_r(:) + real(dp), allocatable :: ptnet_charge(:) + real(dp), allocatable :: mynumel(:) + real(dp) :: mlsi, KSUM + integer :: nats, l, m, lm, info, norbs + integer, allocatable :: ipiv(:) + type(bml_matrix_t) :: zq_bml, zqt_bml + type(bml_matrix_t) :: ptham_bml, ptrho_bml, ptaux_bml + + !Charge perturbation vector. A charge +1 is added to an atomic + !site to compute the response. + if(.not. allocated(chargePertVect))allocate(chargePertVect(nats)) + + !The coulomb potentials will be computed for every perturbation. + allocate(my_coul_forces_k(3,nats)) + allocate(my_coul_forces_r(3,nats)) + allocate(my_coul_pot_k(nats)) + allocate(my_coul_pot_r(nats)) + + allocate(Jacob(nats*nats)) + Jacob = 0.0_dp + + chargePertVect=0.0_dp + do i=1,nats + call gpmdcov_msI("gpmdcov_get_kernel","Constructing response for atom ="//to_string(i),lt%verbose,myRank) + mlsi = mls() + chargePertVect=0.0_dp + chargePertVect(i)=1.0_dp + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,my_coul_forces_r,my_coul_pot_r); + + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%recip_vector,sy%volr,lt%coul_acc,my_coul_forces_k,my_coul_pot_k); + + call gpmdcov_msI("gpmdcov_get_kernel","Time for coulomb "//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + norbs = syprt(ipt)%estr%norbs + + allocate(ptcoul_pot_k(syprt(ipt)%nats)) + allocate(ptcoul_pot_r(syprt(ipt)%nats)) + allocate(ptnet_charge(syprt(ipt)%nats)) + + ptcoul_pot_k = 0.0_dp + ptcoul_pot_r = 0.0_dp + ptnet_charge = 0.0_dp + + !> Get Coulombic potential and charges for the part. + do j=1,gpat%sgraph(ipt)%lsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + ptcoul_pot_k(j) = my_coul_pot_k(jj) + ptcoul_pot_r(j) = my_coul_pot_r(jj) + ptnet_charge(j) = chargePertVect(jj) + enddo + + !Get part(j)\%hamPert given mysyprt(j)\%over, and mysyprt(j)\%pot + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptham_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptrho_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zq_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zqt_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptaux_bml) + + mlsi = mls() + call gpmdcov_msI("gpmdcov_getKernel","Entering prg_get_hscf to construct perturbative ham ...",lt%verbose,myRank) + call prg_get_hscf(ptaux_bml,syprt(ipt)%estr%over,ptham_bml,syprt(ipt)%spindex,& + syprt(ipt)%estr%hindex,tb%hubbardu,ptnet_charge,& + ptcoul_pot_r,ptcoul_pot_k,lt%mdim,lt%threshold) + + call gpmdcov_msI("gpmdcov_get_kernel","Time for H construction "//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + deallocate(ptcoul_pot_k) + deallocate(ptcoul_pot_r) + + call bml_multiply(syprt(ipt)%estr%zmat,syprt(ipt)%estr%evects,zq_bml, 1.0_dp,0.0_dp,lt%threshold) + call bml_transpose(zq_bml,zqt_bml) + call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) + call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) + + ! call prg_canon_response(ptrho_bml,ptham_bml,norbs,beta,syprt(ipt)%estr%evects,& + ! &syprt(ipt)%estr%evals,ef,12,norbs) + + call prg_canon_response(ptrho_bml,ptham_bml,1.0_dp,beta,& + &syprt(ipt)%estr%evals,ef,12,norbs) + + call gpmdcov_msI("gpmdcov_get_kernel","Time for Canonincal Response construction "& + &//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + call bml_multiply(zq_bml,ptrho_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,ptrho_bml,2.0_dp,0.0_dp,0.0_dp) + + call bml_deallocate(ptham_bml) + call bml_deallocate(zq_bml) + call bml_deallocate(zqt_bml) + call bml_deallocate(ptaux_bml) + + mlsi = mls() + allocate(mynumel(10)) + mynumel = 0.0_dp + call prg_get_charges(ptrho_bml, syprt(ipt)%estr%over, syprt(ipt)%estr%hindex, ptnet_charge, mynumel,& + syprt(ipt)%spindex, lt%mdim, lt%threshold) + deallocate(mynumel) + + call gpmdcov_msIII("gpmdcov_get_kernel","Time for getting charges"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + call bml_deallocate(ptrho_bml) + + !Constructing 1D matrix J to prepare for all-gather + do j=1,gpat%sgraph(ipt)%llsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + Jacob((i-1)*nats + jj) = ptnet_charge(j) + enddo + deallocate(ptnet_charge) + enddo + + enddo + + deallocate(my_coul_pot_k) + deallocate(my_coul_pot_r) + deallocate(my_coul_forces_k) + deallocate(my_coul_forces_r) + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(Jacob, nats*nats) + endif +#endif + call prg_wait + + if(.not. allocated(Ker))then + allocate(Ker(nats,nats)) + endif + + lm = 0 + do l=1,nats + do m=1,nats + lm = lm + 1 + if(l == m) then + Ker(m,l) = Jacob(lm) - 1.0_dp + else + Ker(m,l) = Jacob(lm) + endif + ! write(234,*)m,l,Ker(m,l) + enddo + enddo + + ! call bml_print_matrix("JJ",Ker,0,4,0,4) + deallocate(Jacob) + + allocate(work(nats+nats*nats)) + allocate(ipiv(nats)) + + call DGETRF(nats, nats, Ker, nats, ipiv, info) + call DGETRI(nats, Ker, nats, ipiv, work, nats+nats*nats, info) + deallocate(work) + deallocate(ipiv) + + do m = 1,Nats + KSum = 0.D0 + do l = 1,Nats + KSum = KSum + Ker(l,m) + enddo + !write(*,*) ' KSUM ', m, ' = ',KSum + enddo + + end subroutine gpmdcov_getKernel + + + subroutine gpmdcov_getKernel_byBlocks(nats) + use gpmdcov_vars + use gpmdcov_writeout_mod + use prg_response_mod + real(dp), allocatable :: chargePertVect(:) + real(dp), allocatable :: ptcoul_forces_k(:,:) + real(dp), allocatable :: my_coul_forces_k(:,:),my_coul_forces_r(:,:) + real(dp), allocatable :: my_coul_pot_k(:), my_coul_pot_r(:) + real(dp), allocatable :: Jacob(:), work(:), ptcoul_pot_k(:),ptcoul_pot_r(:) + real(dp), allocatable :: ptnet_charge(:) + real(dp), allocatable :: mynumel(:) + real(dp) :: mlsi + integer :: nats, l, m, lm, info, atom, norbs + integer, allocatable :: ipiv(:) + type(bml_matrix_t) :: zq_bml, zqt_bml + type(bml_matrix_t) :: ptham_bml, ptrho_bml, ptaux_bml + + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Entering routine",myRank) + + !Each rank will do its own part +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + !Charge perturbation vector. A charge +1 is added to an atomic + !site to compute the response. + if(.not. allocated(chargePertVect))allocate(chargePertVect(sy%nats)) + + !The coulomb potentials will be computed for every perturbation. + if(.not.allocated(my_coul_forces_k))allocate(my_coul_forces_k(3,sy%nats)) + if(.not.allocated(my_coul_forces_r))allocate(my_coul_forces_r(3,sy%nats)) + if(.not.allocated(my_coul_pot_k))allocate(my_coul_pot_k(sy%nats)) + if(.not.allocated(my_coul_pot_r))allocate(my_coul_pot_r(sy%nats)) + my_coul_pot_k = 0.0_dp + my_coul_pot_r = 0.0_dp + + if(.not. allocated(Jacob)) allocate(Jacob(sy%nats*sy%nats)) + Jacob = 0.0_dp + + chargePertVect=0.0_dp + + do i=1,syprt(ipt)%nats + mlsi = mls() + chargePertVect=0.0_dp + atom = gpat%sgraph(ipt)%core_halo_index(i)+1 + chargePertVect(atom)=1.0_dp + + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Constructing response for atom ="//to_string(atom),lt%verbose,myRank) + + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,my_coul_forces_r,my_coul_pot_r); + + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%recip_vector,sy%volr,lt%coul_acc,my_coul_forces_k,my_coul_pot_k); + + if(.not.allocated(ptcoul_pot_k))allocate(ptcoul_pot_k(syprt(ipt)%nats)) + if(.not.allocated(ptcoul_pot_r))allocate(ptcoul_pot_r(syprt(ipt)%nats)) + if(.not.allocated(ptnet_charge))allocate(ptnet_charge(syprt(ipt)%nats)) + + ptcoul_pot_k = 0.0_dp + ptcoul_pot_r = 0.0_dp + ptnet_charge = 0.0_dp + + !> Get Coulombic potential and charges for the part. + do j=1,gpat%sgraph(ipt)%lsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + ptcoul_pot_k(j) = my_coul_pot_k(jj) + ptcoul_pot_r(j) = my_coul_pot_r(jj) + ptnet_charge(j) = chargePertVect(jj) + enddo + + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for coulomb "//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + norbs = syprt(ipt)%estr%norbs + + !Get part(j)\%hamPert given mysyprt(j)\%over, and mysyprt(j)\%pot + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptham_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptrho_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zq_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zqt_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptaux_bml) + + mlsi = mls() + call gpmdcov_msI("gpmdcov_getKernel_byBlocks","Entering prg_get_hscf to construct perturbative ham ...",lt%verbose,myRank) + call prg_get_hscf(ptaux_bml,syprt(ipt)%estr%over,ptham_bml,syprt(ipt)%spindex,& + syprt(ipt)%estr%hindex,tb%hubbardu,ptnet_charge,& + ptcoul_pot_r,ptcoul_pot_k,lt%mdim,lt%threshold) + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for H construction "//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + deallocate(ptcoul_pot_k) + deallocate(ptcoul_pot_r) + + mlsi = mls() + call bml_multiply(syprt(ipt)%estr%zmat,syprt(ipt)%estr%evects,zq_bml, 1.0_dp,0.0_dp,lt%threshold) + call bml_transpose(zq_bml,zqt_bml) + call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) + call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for tranf to eigen basis " & + &//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + mlsi = mls() + !call prg_canon_response(ptrho_bml,ptham_bml,norbs,23.208882712264995_dp,syprt(ipt)%estr%evects,& + ! &syprt(ipt)%estr%evals,ef,12,norbs) + call prg_canon_response(ptrho_bml,ptham_bml,1.0_dp,beta,& + &syprt(ipt)%estr%evals,ef,12,norbs) + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for Canonincal Response construction " & + &//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + mlsi = mls() + call bml_multiply(zq_bml,ptrho_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,ptrho_bml,2.0_dp,0.0_dp,0.0_dp) + call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for back to canonical basis " & + &//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + call bml_deallocate(ptham_bml) + call bml_deallocate(zq_bml) + call bml_deallocate(zqt_bml) + call bml_deallocate(ptaux_bml) + + mlsi = mls() + allocate(mynumel(10)) + mynumel = 0.0_dp + call prg_get_charges(ptrho_bml, syprt(ipt)%estr%over, syprt(ipt)%estr%hindex, ptnet_charge, mynumel,& + syprt(ipt)%spindex, lt%mdim, lt%threshold) + deallocate(mynumel) + + call gpmdcov_msIII("gpmdcov_get_kernel_byBlocks","Time for getting charges"//to_string(mls() & + &- mlsi)//" ms",lt%verbose,myRank) + call bml_deallocate(ptrho_bml) + + !Constructing 1D matrix J to prepare for all-gather + do j=1,gpat%sgraph(ipt)%llsize + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + Jacob((atom-1)*nats + jj) = ptnet_charge(j) + enddo + deallocate(ptnet_charge) + enddo + + enddo + + if(allocated(ptcoul_pot_k))deallocate(ptcoul_pot_k) + if(allocated(ptcoul_pot_r))deallocate(ptcoul_pot_r) + if(allocated(ptcoul_forces_k))deallocate(ptcoul_forces_k) + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(Jacob, nats*nats) + endif +#endif + call prg_wait + + if(.not. allocated(Ker))then + allocate(Ker(nats,nats)) + endif + + lm = 0 + do l=1,nats + do m=1,nats + lm = lm + 1 + if(l == m) then + Ker(m,l) = Jacob(lm) - 1.0_dp + else + Ker(m,l) = Jacob(lm) + endif + enddo + enddo + + deallocate(Jacob) + + allocate(work(nats+nats*nats)) + allocate(ipiv(nats)) + + call DGETRF(nats, nats, Ker, nats, ipiv, info) + call DGETRI(nats, Ker, nats, ipiv, work, nats+nats*nats, info) + deallocate(work) + deallocate(ipiv) + + ! do m = 1,Nats + ! KSum = 0.D0 + ! do l = 1,Nats + ! KSum = KSum + Ker(l,m) + ! enddo + ! write(*,*) ' KSUM ', m, ' = ',KSum + ! enddo + + end subroutine gpmdcov_getKernel_byBlocks + + subroutine gpmdcov_getKernel_byParts(mysyprt,mysyprtk) + use gpmdcov_vars + use gpmdcov_writeout_mod + use prg_response_mod + real(dp), allocatable :: chargePertVect(:) + real(dp), allocatable :: my_coul_forces_k(:,:),my_coul_forces_r(:,:) + real(dp), allocatable :: my_coul_pot_k(:) + real(dp), allocatable :: my_coul_pot_r(:) + real(dp), allocatable :: work(:) + real(dp), allocatable :: ptcoul_pot_k(:),ptcoul_pot_r(:) + real(dp), allocatable :: ptnet_charge(:) + real(dp) :: mynumel(10) + real(dp) :: mlsi, trdPdMuAO, trp1, mu1, mls_v + integer :: info, atom, coreSize, norbsCore, norbs + integer, allocatable :: ipiv(:) + type(system_type), allocatable, intent(inout) :: mysyprt(:) + type(system_type), allocatable, intent(inout) :: mysyprtk(:) + type(bml_matrix_t) :: zq_bml, zqt_bml + type(bml_matrix_t) :: ptham_bml, ptrho_bml, ptaux_bml + type(bml_matrix_t) :: dPdMuAO_bml,p1_bml,dPdMuAOS_bml,p1S_bml + real(dp), allocatable :: dPdMuAO_dia(:),p1_dia(:),dPdMu(:) + + mls_v = mls() + + if(.not.allocated(my_coul_forces_k))allocate(my_coul_forces_k(3,sy%nats)) + if(.not.allocated(my_coul_forces_r))allocate(my_coul_forces_r(3,sy%nats)) + if(.not.allocated(my_coul_pot_k))allocate(my_coul_pot_k(sy%nats)) + if(.not.allocated(my_coul_pot_r))allocate(my_coul_pot_r(sy%nats)) + if(allocated(mysyprtk))deallocate(mysyprtk) + allocate(mysyprtk(gpat%TotalParts)) + getKernel_byParts_cont = getKernel_byParts_cont + 1 + + call gpmdcov_msI("gpmdcov_get_kernel_byParts", to_string(myRank)//"has "//& + &to_string(partsInEachRank(myRank))//" Parts",lt%verbose,1) + !Each rank will do its own part and keep it. +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + !Charge perturbation vector. A charge +1 is added to an atomic + !site to compute the response. + if(.not.allocated(chargePertVect))allocate(chargePertVect(sy%nats)) + + !The coulomb potentials will be computed for every perturbation. + my_coul_pot_k = 0.0_dp + my_coul_pot_r = 0.0_dp + my_coul_forces_k = 0.0_dp + my_coul_forces_r = 0.0_dp + + coreSize = gpat%sgraph(ipt)%llsize + + if(allocated(mysyprtk(ipt)%estr%ker)) deallocate(mysyprtk(ipt)%estr%ker) + allocate(mysyprtk(ipt)%estr%ker(coreSize,coreSize)) + + mysyprtk(ipt)%estr%ker = 0.0_dp + + do i=1, coreSize + mlsi = mls() + chargePertVect=0.0_dp + atom = gpat%sgraph(ipt)%core_halo_index(i)+1 + chargePertVect(atom)=1.0_dp + + call gpmdcov_msII("gpmdcov_get_kernel_byParts","Constructing response& + & for atom ="//to_string(atom),lt%verbose,myRank) + + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,my_coul_forces_r,my_coul_pot_r); + + call gpmdcov_msII("gpmdcov_get_kernel_byParts","Time for coulomb real & + &"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + mlsi = mls() + + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%recip_vector,sy%volr,lt%coul_acc,my_coul_forces_k,my_coul_pot_k); + + call gpmdcov_msII("gpmdcov_get_kernel_byParts","Time for coulomb recip & + &"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + allocate(ptcoul_pot_k(mysyprt(ipt)%nats)) + allocate(ptcoul_pot_r(mysyprt(ipt)%nats)) + allocate(ptnet_charge(mysyprt(ipt)%nats)) + + ptcoul_pot_k = 0.0_dp + ptcoul_pot_r = 0.0_dp + ptnet_charge = 0.0_dp + + !> Get Coulombic potential and charges for the core part. + do j=1,mysyprt(ipt)%nats + jj = gpat%sgraph(ipt)%core_halo_index(j)+1 + ptcoul_pot_k(j) = my_coul_pot_k(jj) + ptcoul_pot_r(j) = my_coul_pot_r(jj) + ptnet_charge(j) = chargePertVect(jj) + enddo + + + norbs = mysyprt(ipt)%estr%norbs + norbsCore = mysyprt(ipt)%estr%norbsCore + +!!! System -> Cores+halos(0) + + !H = [H_c ] + ! [ H_h ] + + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptham_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptrho_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zq_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,zqt_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,ptaux_bml) + + mlsi = mls() + call gpmdcov_msIII("gpmdcov_getKernel_byParts","Entering prg_get_hscf to& + &construct perturbative ham ...",lt%verbose,myRank) + call prg_get_hscf(ptaux_bml,mysyprt(ipt)%estr%over,ptham_bml,mysyprt(ipt)%spindex,& + mysyprt(ipt)%estr%hindex,tb%hubbardu,ptnet_charge,& + ptcoul_pot_r,ptcoul_pot_k,lt%mdim,lt%threshold) + + + deallocate(ptcoul_pot_r) + deallocate(ptcoul_pot_k) + + call gpmdcov_msIII("gpmdcov_get_kernel_byParts","Time for H construction& + &"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + mlsi = mls() + call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,& + &1.0_dp,0.0_dp,lt%threshold) + call bml_transpose(zq_bml,zqt_bml) + call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) + call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) + call gpmdcov_msIII("gpmdcov_get_kernel_byParts","Time for trasnf to eigenbasis& + &"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + mlsi = mls() + allocate(dPdMu(norbs)) + !call prg_canon_response2_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& + ! &mysyprt(ipt)%estr%evals,ef,12,norbs,lt%threshold) + call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + &norbsCore,beta,mysyprt(ipt)%estr%evects,& + &mysyprt(ipt)%estr%evals,ef,12,norbs) + call bml_get_diagonal(p1_bml,p1_dia) + trP1 = sum(p1_dia(1:norbsCore)) + call gpmdcov_msII("gpmdcov_get_kernel_byParts","Time for Canonincal& + &Response construction "//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,dPdMuAO_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,dPdMuAOS_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norbs,norbs,p1S_bml) + call bml_set_diagonal(dPdMuAO_bml,dPdMu) + + deallocate(dPdMu) + + mlsi = mls() + call bml_multiply(zq_bml,p1_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,p1_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(p1_bml,syprt(ipt)%estr%over,p1S_bml,1.0_dp,0.0_dp,0.0_dp) + + call bml_multiply(zq_bml,dPdMuAO_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,dPdMuAO_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(dPdMuAO_bml,syprt(ipt)%estr%over,dPdMuAOS_bml,1.0_dp,0.0_dp,0.0_dp) + call gpmdcov_msIII("gpmdcov_get_kernel_byParts","Time for trasnf to canonical& + &"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + call bml_get_diagonal(dPdMuAOS_bml,dPdMuAO_dia) + call bml_get_diagonal(p1S_bml,p1_dia) + trP1 = sum(p1_dia(1:norbsCore)) + !write(*,*)"trP1",trP1 + trdPdMuAO = sum(dPdMuAO_dia(1:norbsCore)) + deallocate(dPdMuAO_dia) + deallocate(p1_dia) + if(abs(trdPdMuAO) < 1.0E-12)then + mu1 = 0.0 + else + mu1 = -trP1/trdPdMuAO + endif + !write(*,*)"trP1,trdPdMuAO,mu1",trP1,trdPdMuAO,mu1 + call bml_copy(p1_bml,ptrho_bml) + call bml_add(ptrho_bml,dPdMuAO_bml,2.0_dp,2.0_dp*mu1,lt%threshold) + + call bml_deallocate(ptham_bml) + call bml_deallocate(zq_bml) + call bml_deallocate(zqt_bml) + call bml_deallocate(ptaux_bml) + call bml_deallocate(p1_bml) + call bml_deallocate(dPdMuAO_bml) + call bml_deallocate(dPdMuAOS_bml) + call bml_deallocate(p1S_bml) + + mlsi = mls() + mynumel = 0.0_dp + + !! P1_MO = P1_MO + mu_1*dPdmu -> P1_ao, Tr_core(P1_ao*S) = 0 + !! P1_MO = P1_MO + mu_1*dPdmu -> P1_ao, Tr_core(ZQ*(P1_e + mu_1 dPdMu_e)QZ'*S) = 0 + !! P1_MO = P1_MO + mu_1*dPdmu -> P1_ao, Tr_core(ZQ*P1_e*ZQ'*S) + mu_1*Tr_core(ZQ*dPdMu_e*ZQ'*S) = 0 + !! P1_ao = ZQ*P1_e*ZQ' + mu_1*ZQ*dPdMu_e*QZ' + call prg_get_charges(ptrho_bml, mysyprt(ipt)%estr%over,& + &mysyprt(ipt)%estr%hindex, ptnet_charge, mynumel,& + mysyprt(ipt)%spindex, norbs, lt%threshold) + + call bml_deallocate(ptrho_bml) + call gpmdcov_msII("gpmdcov_get_kernel_byParts","Time for getting & + &charges"//to_string(mls() - mlsi)//" ms",lt%verbose,myRank) + + ! do m = 1,coreSize + ! KSum = 0.0_dp + ! do l = 1,coreSize + ! KSum = KSum + ptnet_charge(l) + ! enddo + ! write(*,*) ' JSUM ', m, ' = ',KSum + ! enddo + + !Constructing J to prepare for K + do j=1,coreSize + mysyprtk(ipt)%estr%ker(j,i) = ptnet_charge(j) + if(i == j)mysyprtk(ipt)%estr%ker(j,i) = mysyprtk(ipt)%estr%ker(j,i) - 1.0_dp + !Regularized Kernel + !if(i == j)mysyprtk(ipt)%estr%ker(j,i) = mysyprtk(ipt)%estr%ker(j,i) - 1.2_dp + enddo + deallocate(ptnet_charge) + call bml_deallocate(ptrho_bml) + + enddo + + if(allocated(work))deallocate(work);allocate(work(coreSize+coreSize*coreSize)) + if(allocated(ipiv))deallocate(ipiv);allocate(ipiv(coreSize)) + call DGETRF(coreSize, coreSize, mysyprtk(ipt)%estr%ker, coreSize, ipiv, info) + call DGETRI(coreSize, mysyprtk(ipt)%estr%ker, coreSize, ipiv,& + & work, coreSize+coreSize*coreSize, info) + deallocate(work) + deallocate(ipiv) + + ! do m = 1,coreSize + ! KSum = 0.0_dp + ! do l = 1,coreSize + ! KSum = KSum + mysyprtk(ipt)%estr%ker(l,m) + ! enddo + ! write(*,*) ' KSUM ', m, ' = ',KSum + ! enddo + +! Hack to use the identity (to see the effect of low rank updates) + !mysyprtk(ipt)%estr%ker = 0.0_dp + !write(*,*)"WARNING!!!!!!! using Identity as precond" + !do m = 1,coreSize + ! mysyprtk(ipt)%estr%ker(m,m) = -0.2_dp + !enddo + enddo + + deallocate(my_coul_forces_k) + deallocate(my_coul_forces_r) + deallocate(my_coul_pot_k) + deallocate(my_coul_pot_r) + + call gpmdcov_msI("gpmdcov_get_kernel_byParts - Rank "//to_string(myRank) ,"Time for get_kernel_byParts & + &"//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) + + + + end subroutine gpmdcov_getKernel_byParts + + + subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Res) + use gpmdcov_vars + use gpmdcov_writeout_mod + use prg_response_mod + use gpmdcov_allocation_mod + implicit none + integer :: myi,myj,myjj,irank,mRanks + integer :: norbsCore,norbs + integer, intent(in) :: maxRanks + integer :: natsCore,natsCoreHalo + integer, allocatable :: maxCoresAmongParts(:) + integer :: maxCoresAmongPartsAndRanks + real(dp), allocatable, intent(in) :: myqn(:), myn(:) + real(dp), allocatable :: myqnPart(:), mynPart(:) + real(dp), allocatable :: dr(:),K0Res(:),K0ResPart(:,:),dr_save(:,:) + real(dp), allocatable :: vi(:,:), v(:) + real(dp), allocatable :: my_coul_pot(:),ptcoul_forces_k(:,:) + real(dp), allocatable :: my_coul_forces_k(:,:),my_coul_forces_r(:,:) + real(dp), allocatable :: ptcoul_forces_r(:,:), my_coul_pot_k(:) + real(dp), allocatable :: my_coul_pot_r(:),chargePertVect(:) + real(dp), allocatable :: ptcoul_pot_k(:) + real(dp), allocatable :: ptcoul_pot_r(:) + real(dp), allocatable :: ptnet_charge(:) + real(dp), allocatable :: v_core_i(:,:,:) + real(dp), allocatable :: dPdMu(:), IdK0Res(:) + real(dp), allocatable :: q1(:,:),dqdmu(:,:) + real(dp), allocatable :: f(:),c_i(:),ff(:,:,:) + real(dp), allocatable :: oij(:,:),mMat(:,:) + real(dp), allocatable :: vRank(:) + real(dp), intent(inout), allocatable :: KK0Res(:) + real(dp) :: trP1(1), trdPdMu(1) + real(dp) :: mu1_global, error + type(bml_matrix_t) :: ptham_bml, ptaux_bml + type(bml_matrix_t) :: zq_bml,zqt_bml + type(bml_matrix_t) :: p1_bml,dPdMu_bml + type(system_type), allocatable, intent(inout) :: mysyprt(:) + type(system_type), allocatable, intent(inout) :: mysyprtk(:) + real(dp) :: mynumel(10), mls_v + real(dp), allocatable :: work(:),auxVect(:) + integer :: info + integer, allocatable :: ipiv(:) + + call gpmdcov_msI("gpmdcov_rankN_update_byParts","Updating the Kernel",lt%verbose,myRank) + rankN_update_byParts_cont = rankN_update_byParts_cont + 1 + + mls_v = mls() + + if(.not.allocated(K0Res))allocate(K0Res(sy%nats)) + + K0Res = 0.0_dp + + !This will be used to capture the maximun number of core atoms + !among all the parts/subsystems for the present MPI rank. + allocate(maxCoresAmongParts(nRanks)) + maxCoresAmongParts = 0 +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + natsCore = gpat%sgraph(ipt)%llsize + maxCoresAmongParts(myRank) = max(maxCoresAmongParts(myRank),natsCore) + enddo +!stop + call gpmdcov_reallocate_realMat(K0ResPart,maxCoresAmongParts(myRank),partsInEachRank(myRank)) + + K0ResPart = 0.0_dp + + !In this loop we will compute K0Res which is the product of the + !Preconditioner K with the residue q(n) - n +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + + natsCore = gpat%sgraph(ipt)%llsize + + call gpmdcov_reallocate_realVect(myqnPart,natsCore) + call gpmdcov_reallocate_realVect(mynPart,natsCore) + + !Get old charges and nguess for the part + do myj=1,natsCore + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + mynPart(myj) = myn(myjj) + myqnPart(myj) = myqn(myjj) + enddo + + K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) + + !Expand K0resPart into K0Res + do myj=1,natsCore + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + K0Res(myjj) = K0ResPart(myj,iptt) + enddo + deallocate(myqnPart) + deallocate(mynPart) + + enddo + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(K0Res,sy%nats) + call prg_sumIntReduceN(maxCoresAmongParts,nRanks) + endif + call prg_wait() +#endif + + !This is the maximun core atoms among "all" of the parts/subsystems + maxCoresAmongPartsAndRanks = maxval(maxCoresAmongParts) + + allocate(dr(sy%nats)) + + if(.not.allocated(my_coul_forces_k))allocate(my_coul_forces_k(3,sy%nats)) + if(.not.allocated(my_coul_forces_r))allocate(my_coul_forces_r(3,sy%nats)) + if(.not.allocated(my_coul_pot_k))allocate(my_coul_pot_k(sy%nats)) + if(.not.allocated(my_coul_pot_r))allocate(my_coul_pot_r(sy%nats)) + my_coul_pot_k = 0.0_dp + my_coul_pot_r = 0.0_dp + my_coul_forces_k = 0.0_dp + my_coul_forces_r = 0.0_dp + + dr = K0Res + + mRanks = maxRanks !Number of total rank updates + allocate(vi(sy%nats,mRanks)) + allocate(dr_save(sy%nats,mRanks)) + vi = 0.0_dp + allocate(v_core_i(maxCoresAmongPartsAndRanks,partsInEachRank(myRank),mRanks)) + v_core_i = 0.0_dp + allocate(c_i(mRanks)) + c_i = 0.0_dp + allocate(ff(maxCoresAmongPartsAndRanks,partsInEachRank(myRank),mRanks)) + ff = 0.0_dp + + !Here we enter the loop for the rank updates (do not confuse with MPI rank) + do irank = 1, mRanks + vi(:,irank) = dr/norm2(dr) + !Gram-Schmidt OrthoNormalization + if(irank > 1)then + do kk = 1,irank-1 + vi(:,irank) = vi(:,irank) - DOT_PRODUCT(vi(:,kk),vi(:,irank))*vi(:,kk) + enddo + vi(:,irank) = vi(:,irank)/norm2(vi(:,irank)) + endif + chargePertVect = vi(:,irank) + + call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& + nl%nnIz,nl%nrnnlist,nl%nnType,my_coul_forces_r,my_coul_pot_r); + + call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& + ,chargePertVect,tb%hubbardu,sy%lattice_vector,& + sy%recip_vector,sy%volr,lt%coul_acc,my_coul_forces_k,my_coul_pot_k); + + call gpmdcov_reallocate_realMat(q1,maxCoresAmongPartsAndRanks,partsInEachRank(myRank)) + call gpmdcov_reallocate_realMat(dqdmu,maxCoresAmongPartsAndRanks,partsInEachRank(myRank)) + + trP1 = 0.0_dp; trdPdMu = 0.0_dp + +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + norbs = mysyprt(ipt)%estr%norbs + + !We obtain the atoms in the Core+Halo + !Can also be obtained from gpat%sgraph(ipt)%lsize + natsCoreHalo = mysyprt(ipt)%nats + natsCore = gpat%sgraph(ipt)%llsize + + call gpmdcov_reallocate_realVect(ptcoul_pot_k,natsCoreHalo) + call gpmdcov_reallocate_realVect(ptcoul_pot_r,natsCoreHalo) + call gpmdcov_reallocate_realVect(ptnet_charge,natsCoreHalo) + + ptcoul_pot_k = 0.0_dp + ptcoul_pot_r = 0.0_dp + ptnet_charge = 0.0_dp + + !Get Coulombic potential and charges for the part (C+H) + do myj=1,natsCoreHalo + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + ptcoul_pot_k(myj) = my_coul_pot_k(myjj) + ptcoul_pot_r(myj) = my_coul_pot_r(myjj) + ptnet_charge(myj) = chargePertVect(myjj) + enddo + + !Extract the perturbation over the core part only + do myj=1,natsCore + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + v_core_i(myj,iptt,irank) = vi(myjj,irank) + enddo + + !COMMENT: MAYBE v_core_i should not be normalized!!!!!! + + !Computing the perturbative Hamiltonian (H1) from the coulombic perturbations + !generated by dr=K0*res. At the C+H level + call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) + call gpmdcov_reallocate_denseBmlRealMat(ptaux_bml,norbs) + !ptaux_bml corresponds to H0 which is 0 in this case. + call prg_get_hscf_v2(ptaux_bml,mysyprt(ipt)%estr%over,ptham_bml,mysyprt(ipt)%spindex,& + mysyprt(ipt)%estr%hindex,tb%hubbardu,ptnet_charge,& + ptcoul_pot_r,ptcoul_pot_k,norbs,lt%threshold) + + !Compute transformations ZQ and (ZQ)^t transformation that takes from the canonical nonorthogonal + !to the orthogonal eigenbasis. + call gpmdcov_reallocate_denseBmlRealMat(zq_bml,norbs) + call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,1.0_dp,0.0_dp,lt%threshold) + call gpmdcov_reallocate_denseBmlRealMat(zqt_bml,norbs) + call bml_transpose(zq_bml,zqt_bml) + + !Take H1 to the ortho-eigen basis set. + call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,0.0_dp) + + !Construct the "bare" response P1 and the derivative with respect to the + !chemical potential (dPdMu). Everything in the ortho-eigen basis set + call gpmdcov_reallocate_realVect(dPdMu,norbs) + norbsCore = mysyprt(ipt)%estr%norbsCore + call gpmdcov_reallocate_denseBmlRealMat(p1_bml,norbs) + + !call prg_canon_response2_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& + ! &mysyprt(ipt)%estr%evals,ef,12,norbs,lt%threshold) + call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + &norbsCore,beta,mysyprt(ipt)%estr%evects,& + &mysyprt(ipt)%estr%evals,ef,12,norbs) + + !At this point ptham is not needed anymore + if(bml_allocated(ptham_bml)) call bml_deallocate(ptham_bml) + + !Transform P1 back to the nonortho-canonical basis set. + call bml_multiply(zq_bml,p1_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,p1_bml,2.0_dp,0.0_dp,0.0_dp) + + !Since dPdMu is represented by a vector, we will convert it into a BML matrix + call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) + call bml_set_diagonal(dPdMu_bml,dPdMu) + + !Transform dPdMu back to the nonortho-canonical basis set + call bml_multiply(zq_bml,dPdMu_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) + call bml_multiply(ptaux_bml,zqt_bml,dPdMu_bml,2.0_dp,0.0_dp,0.0_dp) + + !At this point ZQ and ZQt are not needed anymore + call bml_deallocate(zq_bml) + call bml_deallocate(zqt_bml) + + !Here we compute the charges response (q1) from P1 and we store it on + !a vector q1 that stores all the previous q1s from past iranks iterations + !We also compute the partial trace contribution (trP1) from this mpi + !execution and the current part (ipt). + ptnet_charge = 0.0_dp + mynumel = 0.0_dp + call prg_get_charges(p1_bml, mysyprt(ipt)%estr%over,& + &mysyprt(ipt)%estr%hindex, ptnet_charge, mynumel,& + mysyprt(ipt)%spindex, norbs, lt%threshold) + q1(:,iptt) = ptnet_charge(1:natsCore) + trP1 = trP1 + sum(ptnet_charge(1:natsCore)) + call bml_deallocate(p1_bml) + + !Here we compute the charges response (dqdmu) from dPdMu and we store + !them on a matrix dqdmu that stores all the previous dqdmus from past + !irank iterations. + !We also compute the partial trace contribution (trdPdMu) from this node + !and the current part (ipt). + ptnet_charge = 0.0_dp + call prg_get_charges(dpdmu_bml, mysyprt(ipt)%estr%over,& + &mysyprt(ipt)%estr%hindex, ptnet_charge, mynumel,& + mysyprt(ipt)%spindex, norbs, lt%threshold) + dqdmu(:,iptt) = ptnet_charge(1:natsCore) + trdPdMu = trdPdMu + sum(ptnet_charge(1:natsCore)) + call bml_deallocate(dPdMu_bml) + call bml_deallocate(ptaux_bml) + deallocate(ptnet_charge) + + enddo !End of the loop over the parts + + !We will gather all the partial traces from all the MPI Ranks. +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(trP1, 1) + call prg_sumRealReduceN(trdPdMu, 1) + endif +#endif + + mu1_Global = -trP1(1)/trdPdMu(1) + q1 = q1 + mu1_Global*dqdmu + + call gpmdcov_reallocate_realVect(f,maxCoresAmongPartsAndRanks) + !deallocate(maxCoresAmongParts) + f = 0.0_dp + dr = 0.0_dp +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + natsCore = gpat%sgraph(ipt)%llsize + + f = q1(:,iptt) - v_core_i(:,iptt,irank) + ff(:,iptt,irank) = MATMUL(syprtk(ipt)%estr%ker,f(1:natsCore)) + c_i(irank) = c_i(irank) + DOT_PRODUCT(ff(1:natsCore,iptt,irank),K0ResPart(1:natsCore,iptt)) + do myj=1,natsCore + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + dr(myjj) = ff(myj,iptt,irank) + enddo + enddo + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(dr, sy%nats) + endif +#endif + dr_save(:,iRank) = dr + + enddo !Rank enddo + + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(c_i, mRanks) + endif +#endif + + allocate(auxVect(mRanks*mRanks)) + auxVect = 0.0_dp + do myi = 1,mRanks + do myj = 1,mRanks + do iptt=1,partsInEachRank(myRank) + auxVect((myi-1)*mRanks + myj) = & + &auxVect((myi-1)*mRanks + myj) + DOT_PRODUCT(ff(:,iptt,myi),ff(:,iptt,myj)) + enddo + enddo + enddo + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(auxVect, mRanks*mRanks) + endif +#endif + + allocate(oij(mRanks,mRanks)) + allocate(mMat(mRanks,mRanks)) + do myi = 1,mRanks + do myj = 1,mRanks + oij(myi,myj) = auxVect((myi-1)*mRanks + myj) + enddo + enddo + + mMat = oij + + if(allocated(work))deallocate(work);allocate(work(mRanks+mRanks*mRanks)) + if(allocated(ipiv))deallocate(ipiv);allocate(ipiv(mRanks)) + call DGETRF(mRanks,mRanks,mMat,mRanks,ipiv,info) + call DGETRI(mRanks,mMat,mRanks,ipiv,work,mRanks+mRanks*mRanks,info) + deallocate(work) + deallocate(ipiv) + + !Reconstruct the full v from the v_core_is + allocate(vRank(sy%nats)) + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) + do myj=1,gpat%sgraph(ipt)%llsize + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + vRank(myjj) = v_core_i(myj,iptt,mRanks) + enddo + enddo + + + +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(vRank, sy%nats) + endif +#endif + + call gpmdcov_reallocate_realVect(KK0Res,sy%nats) + call gpmdcov_reallocate_realVect(IdK0Res,sy%nats) + KK0Res = 0.0_dp + IdK0Res = 0.0_dp + do myi = 1,mRanks + do myj = 1,mRanks + KK0Res = KK0Res + vi(:,myi)*mMat(myi,myj)*c_i(myj) + IdK0Res = IdK0Res + dr_save(:,myi)*mMat(myi,myj)*c_i(myj) + enddo + enddo + + error = norm2(IdK0Res - K0Res)/norm2(K0Res) + if(myRank == 1)write(*,*)"Error Rank-Update",error,mRanks + + !do myi = 1,mRanks + ! do myj = 1,mRanks + ! write(*,*) DOT_PRODUCT(dr_save(:,myi),dr_save(:,myj)) + ! enddo + ! enddo + + + !Deallocations + if(allocated(maxCoresAmongParts))deallocate(maxCoresAmongParts) + if(allocated(myqnPart))deallocate(myqnPart) + if(allocated(mynPart))deallocate(mynPart) + if(allocated(dr))deallocate(dr) + if(allocated(v))deallocate(v) + if(allocated(my_coul_pot))deallocate(my_coul_pot) + if(allocated(ptcoul_forces_k))deallocate(ptcoul_forces_k) + if(allocated(ptcoul_forces_r))deallocate(ptcoul_forces_r) + if(allocated(my_coul_pot_k))deallocate(my_coul_pot_k) + if(allocated(chargePertVect))deallocate(chargePertVect) + if(allocated(ptcoul_pot_k))deallocate(ptcoul_pot_k) + if(allocated(ptcoul_pot_r))deallocate(ptcoul_pot_r) + if(allocated(ptnet_charge))deallocate(ptnet_charge) + if(allocated(v_core_i))deallocate(v_core_i) + if(allocated(dPdMu))deallocate(dPdMu) + if(allocated(IdK0Res))deallocate(IdK0Res) + if(allocated(q1))deallocate(q1) + if(allocated(dqdmu))deallocate(dqdmu) + if(allocated(f))deallocate(f) + if(allocated(c_i))deallocate(c_i) + if(allocated(ff))deallocate(ff) + if(allocated(oij))deallocate(oij) + if(allocated(mMat))deallocate(mMat) + if(allocated(vRank))deallocate(vRank) + if(allocated(dr_save))deallocate(dr_save) + if(allocated(vi))deallocate(vi) + if(allocated(maxCoresAmongParts))deallocate(maxCoresAmongParts) + if(allocated(my_coul_pot_k))deallocate(my_coul_pot_k) + if(allocated(my_coul_pot_r))deallocate(my_coul_pot_r) + if(allocated(my_coul_forces_k))deallocate(my_coul_forces_k) + if(allocated(my_coul_forces_r))deallocate(my_coul_forces_r) + if(allocated(K0Res))deallocate(K0Res) + if(allocated(K0ResPart))deallocate(K0ResPart) + + call gpmdcov_msI("gpmdcov_get_kernel_byParts","Time for get_kernel_byParts & + &"//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) + + end subroutine gpmdcov_rankN_update_byParts + + + subroutine gpmdcov_applyKernel(my_nguess,my_chargesOld,mysyprtk,kernelTimesRes) + use gpmdcov_vars + implicit none + real(dp), allocatable, intent(inout) :: kernelTimesRes(:) + real(dp), allocatable, intent(in) :: my_nguess(:) + real(dp), allocatable, intent(in) :: my_chargesOld(:) + real(dp), allocatable :: nguessPart(:), chargesOldPart(:) + real(dp), allocatable :: kernelTimesResPart(:) + integer :: nats, myj, myjj + type(system_type), allocatable, intent(in) :: mysyprtk(:) + + call gpmdcov_msI("gpmdcov_applyKernel","Applying the kernel...",lt%verbose,myRank) + + !> Get Coulombic potential and charges for the part. + nats = size(my_nguess,dim=1) + if(.not. allocated(kernelTimesRes)) allocate(kernelTimesRes(nats)) + kernelTimesRes = 0.0_dp + +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + allocate(kernelTimesResPart(gpat%sgraph(ipt)%llsize)) + allocate(nguessPart(gpat%sgraph(ipt)%llsize)) + allocate(chargesOldPart(gpat%sgraph(ipt)%llsize)) + + !Collapse old charges and nguess for the part + do myj=1,gpat%sgraph(ipt)%llsize + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + chargesOldPart(myj) = my_chargesOld(myjj) + nguessPart(myj) = my_nguess(myjj) + enddo + kernelTimesResPart = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) + + !Expand + do myj=1,gpat%sgraph(ipt)%llsize + myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 + kernelTimesRes(myjj) = kernelTimesResPart(myj) + enddo + + deallocate(kernelTimesResPart) + deallocate(nguessPart) + deallocate(chargesOldPart) + + enddo +#ifdef DO_MPI + if (getNRanks() .gt. 1) then + call prg_sumRealReduceN(kernelTimesRes, nats) + endif + call prg_wait +#endif + +call gpmdcov_msI("gpmdcov_applyKernel","End of Applying the kernel...",lt%verbose,myRank) + end subroutine gpmdcov_applyKernel + +end module gpmdcov_kernel_mod diff --git a/examples/gpmdk/src/gpmdcov_langevin.F90 b/examples/gpmdk/src/gpmdcov_langevin.F90 new file mode 100644 index 00000000..32d59a85 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_langevin.F90 @@ -0,0 +1,261 @@ +module gpmdcov_langevin_mod + private + + integer, parameter :: dp = kind(1.0d0) + + real(dp), parameter :: F2V = 0.01602176487_dp/1.660548782_dp + + real(dp), parameter :: KE2T = 1.0_dp/0.000086173435_dp + + real(dp), parameter :: MVV2KE = 166.0538782_dp/1.602176487_dp + + real(dp), parameter :: twopi = 8.0_dp*atan(1.0_dp) + + public :: gpmdcov_uniform_to_normal, LangevinDVGoga, LangevinDVSivaOne, LangevinDVSivaTwo, LangevinDXGoga + +contains + + !> Convert uniform randomly distributed rands to standard-normally distributed rands + !> Requires an input 3-d array of rands with length 2 in last dim + !> Returns the normally distributed rands in the first position of the last dim + !! \param rands 3-d array of rands with length 2 in last dim + + subroutine gpmdcov_uniform_to_normal(rands) + + real, intent(inout) :: rands(:,:,:) + real, allocatable :: amp(:,:), tinies(:) + logical, allocatable :: tinymask(:,:) + real :: smallest + + if(.not.allocated(amp))then + allocate(amp(size(rands,dim=1),size(rands,dim=2))) + allocate(tinymask(size(amp,dim=1),size(amp,dim=2))) + endif + + !***Important*** + !Replace rands that are too small for the log() + !If this is left out then eventually the program will + ! crash + smallest = tiny(smallest) + tinymask = rands(:,:,1).lt.smallest + if(any(tinymask))then + tinies = pack(rands(:,:,1),tinymask) + do while(any(tinies.lt.smallest)) + call random_number(tinies) + end do + rands(:,:,1) = unpack(tinies,tinymask,rands(:,:,1)) + endif + + !Generate normally distributed rands using Box-Muller + amp = log(rands(:,:,1)) + amp = sqrt(-2.0 * amp) + + rands(:,:,1) = amp(:,:)*cos(real(twopi)*rands(:,:,2)) + rands(:,:,2) = amp(:,:)*sin(real(twopi)*rands(:,:,2)) + + end subroutine gpmdcov_uniform_to_normal + + !> Half Verlet Langevin integration. Step one, before coordinate update. + !! \param mass Mass of every atom in the system. + !! \param FTOT Total force on every atom. + !! \param timestep Time step for Verlet integration (fs). + !! \param VX Velocities for the x direction. + !! \param VY Velocities for the y direction. + !! \param VZ Velocities for the z direction. + !! \param f Friction constant + !! \param T Temperature (K) + subroutine LangevinDVGoga(mass,FTOT,timestep,V,DV,gamma,T,rands) + + real(dp), intent(in) :: mass(:) + real(dp), intent(in) :: FTOT(:,:) + real(dp), intent(in) :: timestep + real(dp), intent(inout) :: V(:,:) + real(dp), intent(out) :: DV(:,:) + real(dp), intent(in) :: gamma + real(dp), intent(in) :: T + real, intent(in) :: rands(:,:,:) + integer :: i + real(dp) :: f + + f = 1._dp - exp(-gamma*timestep) + + + do i = 1,3 + V(i,:) = V(i,:) + timestep*(F2V*FTOT(i,:)/mass(:)) + DV(i,:) = -f*V(i,:) + sqrt(T/KE2T/MVV2KE*f*(2.0_dp-f)/mass(:))*rands(:,i,1) + end do + !DVX(:) = -f*VX(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,1,1) + !DVY(:) = -f*VY(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,2,1) + !DVZ(:) = -f*VZ(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,3,1) + !VX(:) = VX(:) + 0.5_dp*timestep*(F2V*FTOT(1,:)/mass(:)) + !VY(:) = VY(:) + 0.5_dp*timestep*(F2V*FTOT(2,:)/mass(:)) + !VZ(:) = VZ(:) + 0.5_dp*timestep*(F2V*FTOT(3,:)/mass(:)) + + end subroutine LangevinDVGoga + + subroutine LangevinDVSivaOne(mass,FTOT,timestep,V,gamma,T,rands) + + real(dp), intent(in) :: mass(:) + real(dp), intent(in) :: FTOT(:,:) + real(dp), intent(in) :: timestep + real(dp), intent(inout) :: V(:,:) + real(dp), intent(in) :: gamma + real(dp), intent(in) :: T + real, intent(in) :: rands(:,:,:) + real(dp) :: a, b, c + integer :: i + + ! real(dp) :: alpha, beta, norm_alpha, norm_beta, cdf_alpha, cdf_beta, sigfac + ! real, allocatable :: thresholded_rands(:,:) + ! logical, allocatable :: mask1(:,:), mask2(:,:) + + ! if(.not.allocated(thresholded_rands))then + ! allocate(thresholded_rands(size(rands,dim=1),size(rands,dim=2))) + ! allocate(mask1(size(rands,dim=1),size(rands,dim=2))) + ! allocate(mask2(size(rands,dim=1),size(rands,dim=2))) + ! endif + + ! alpha = -6.0_dp + ! beta = 6.0_dp + + ! mask1 = rands(:,:,1).lt.alpha.or.rands(:,:,1).gt.beta + ! mask2 = rands(:,:,2).lt.alpha.or.rands(:,:,2).gt.beta + + ! if(ANY(mask1.or.mask2))then + ! write(*,*)"Langevin random velocity and correction are both out of range" + ! stop + ! endif + + ! thresholded_rands = merge(rands(:,:,2),rands(:,:,1),mask1) + + ! norm_alpha = exp(-alpha*alphs/2.0_dp)/sqrt(twopi) + ! norm_beta = exp(-beta*beta/2.0_dp)/sqrt(twopi) + ! cdf_alpha = (1. + erf(alpha/sqrt(2.0_dp)))/2.0_dp + ! cdf_beta = (1. + erf(beta/sqrt(2.0_dp)))/2.0_dp + ! sigfac = sqrt(1.0_dp - (beta*norm_beta-alpha*norm_alpha)/(cdf_beta - cdf_alpha) - ((norm_beta-norm_alpha)/(cdf_beta-cdf_alpha))) + + a = exp(-gamma*timestep) + c = 2.0_dp/gamma/timestep + b = sqrt(c*tanh(1.0_dp/c)) + + do i = 1,3 + !V(i,:) = sqrt(a)*V(i,:) + sqrt((1.0_dp-a)*T/KE2T/MVV2KE/mass(:))*thresholded_rands(:,i)/sigfac +! do j = 1,size(mass) +! write(*,*)mass(j),rands(j,i,1) +! V(i,j) = sqrt(a)*V(i,j) + sqrt((1.0_dp-a)*T/KE2T/MVV2KE/mass(j))*rands(j,i,1) +! enddo + V(i,:) = sqrt(a)*V(i,:) + sqrt((1.0_dp-a)*T/KE2T/MVV2KE/mass(:))*rands(:,i,1) + V(i,:) = V(i,:) + b*timestep/2.0_dp*(F2V*FTOT(i,:)/mass(:)) + end do + + !DVX(:) = -f*VX(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,1,1) + !DVY(:) = -f*VY(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,2,1) + !DVZ(:) = -f*VZ(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,3,1) + !VX(:) = VX(:) + 0.5_dp*timestep*(F2V*FTOT(1,:)/mass(:)) + !VY(:) = VY(:) + 0.5_dp*timestep*(F2V*FTOT(2,:)/mass(:)) + !VZ(:) = VZ(:) + 0.5_dp*timestep*(F2V*FTOT(3,:)/mass(:)) + + end subroutine LangevinDVSivaOne + + subroutine LangevinDVSivaTwo(mass,FTOT,timestep,V,gamma,T,rands) + + real(dp), intent(in) :: mass(:) + real(dp), intent(in) :: FTOT(:,:) + real(dp), intent(in) :: timestep + real(dp), intent(inout) :: V(:,:) + real(dp), intent(in) :: gamma + real(dp), intent(in) :: T + real, intent(in) :: rands(:,:,:) + real(dp) :: a, b, c + integer :: i + ! real(dp) :: alpha, beta, norm_alpha, norm_beta, cdf_alpha, cdf_beta, sigfac + ! real, allocatable :: thresholded_rands(:,:) + ! logical, allocatable :: mask1(:,:), mask2(:,:) + + ! if(.not.allocated(thresholded_rands))then + ! allocate(thresholded_rands(size(rands,dim=1),size(rands,dim=2))) + ! allocate(mask1(size(rands,dim=1),size(rands,dim=2))) + ! allocate(mask2(size(rands,dim=1),size(rands,dim=2))) + ! endif + + ! alpha = -6.0_dp + ! beta = 6.0_dp + + ! mask1 = rands(:,:,1).lt.alpha.or.rands(:,:,1).gt.beta + ! mask2 = rands(:,:,2).lt.alpha.or.rands(:,:,2).gt.beta + + ! if(ANY(mask1.or.mask2))then + ! write(*,*)"Langevin random velocity and correction are both out of range" + ! stop + ! endif + + ! thresholded_rands = merge(rands(:,:,2),rands(:,:,1),mask1) + + ! norm_alpha = exp(-alpha*alphs/2.0_dp)/sqrt(twopi) + ! norm_beta = exp(-beta*beta/2.0_dp)/sqrt(twopi) + ! cdf_alpha = (1. + erf(alpha/sqrt(2.0_dp)))/2.0_dp + ! cdf_beta = (1. + erf(beta/sqrt(2.0_dp)))/2.0_dp + ! sigfac = sqrt(1.0_dp - (beta*norm_beta-alpha*norm_alpha)/(cdf_beta - cdf_alpha) - ((norm_beta-norm_alpha)/(cdf_beta-cdf_alpha))) + + a = exp(-gamma*timestep) + c = 2.0_dp/gamma/timestep + b = sqrt(c*tanh(1.0_dp/c)) + + do i = 1,3 + V(i,:) = V(i,:) + b*timestep/2.0_dp*(F2V*FTOT(i,:)/mass(:)) + !V(i,:) = sqrt(a)*V(i,:) + sqrt((1.0_dp-a)*T/KE2T/MVV2KE/mass(:))*thresholded_rands(:,i)/sigfac + V(i,:) = sqrt(a)*V(i,:) + sqrt((1.0_dp-a)*T/KE2T/MVV2KE/mass(:))*rands(:,i,1) + end do + + !DVX(:) = -f*VX(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,1,1) + !DVY(:) = -f*VY(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,2,1) + !DVZ(:) = -f*VZ(:) + sqrt(T/KE2T*f*(2.0_dp-f)/mass(:))*rands(:,3,1) + !VX(:) = VX(:) + 0.5_dp*timestep*(F2V*FTOT(1,:)/mass(:)) + !VY(:) = VY(:) + 0.5_dp*timestep*(F2V*FTOT(2,:)/mass(:)) + !VZ(:) = VZ(:) + 0.5_dp*timestep*(F2V*FTOT(3,:)/mass(:)) + + end subroutine LangevinDVSivaTwo + + !> Update atomic positions. + !! \param origin Coordinate origin. + !! \param lattice_vector Lattice vectors of the system. + !! \param timestep Time step for Verlet integration. + !! \param VX Velocities for the x direction. + !! \param VY Velocities for the y direction. + !! \param VZ Velocities for the z direction. + !! \param coordinate Coordinates of every atom in the system. + subroutine LangevinDXGoga(origin,lattice_vector,timestep,VX,VY,VZ,DVX,DVY,DVZ,coordinate) + implicit none + real(dp), intent(in) :: origin(:),lattice_vector(:,:) + real(dp), intent(in) :: VX(:), VY(:), VZ(:), timestep + real(dp), intent(in) :: DVX(:), DVY(:), DVZ(:) + real(dp), intent(inout) :: coordinate(:,:) + integer :: i + + do i=1,size(coordinate,dim=2) + coordinate(1,i) = coordinate(1,i) + timestep*(VX(i)+0.5_dp*DVX(i)) + if(coordinate(1,i)-origin(1).gt.lattice_vector(1,1))then + coordinate(1,i) = coordinate(1,i) - lattice_vector(1,1) + endif + if(coordinate(1,i)-origin(1).lt.0.0_dp)then + coordinate(1,i) = coordinate(1,i) + lattice_vector(1,1) + endif + coordinate(2,i) = coordinate(2,i) + timestep*(VY(i)+0.5_dp*DVY(i)) + if(coordinate(2,i)-origin(2).gt.lattice_vector(2,2))then + coordinate(2,i) = coordinate(2,i) - lattice_vector(2,2) + endif + if(coordinate(2,i)-origin(2).lt.0.0_dp)then + coordinate(2,i) = coordinate(2,i) + lattice_vector(2,2) + endif + coordinate(3,i) = coordinate(3,i) + timestep*(VZ(i)+0.5_dp*DVZ(i)) + if(coordinate(3,i)-origin(3).gt.lattice_vector(3,3))then + coordinate(3,i) = coordinate(3,i) - lattice_vector(3,3) + endif + if(coordinate(3,i)-origin(3).lt.0.0_dp)then + coordinate(3,i) = coordinate(3,i) + lattice_vector(3,3) + endif + enddo + + end subroutine LangevinDXGoga + +end module gpmdcov_langevin_mod diff --git a/examples/gpmdk/src/gpmdcov_latte.F90 b/examples/gpmdk/src/gpmdcov_latte.F90 new file mode 100644 index 00000000..19a74ad2 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_latte.F90 @@ -0,0 +1,149 @@ +!> Module for calling original latte routines +!! \brief This module will be used to call latte routines +!! using the origial code +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_latte_mod + + use gpmdcov_vars + use prg_system_mod + ! Latte modules +#ifdef USE_LATTE + use setuparray + use prg_system_mod + use latteparser + use constants_mod + use bml + use nonoarray + use NEBLISTARRAY + use univarray +#endif + +public :: init_latte + +contains + + !> Initialize LATTE variables by passing + !! the total system. + !! \brief This will read the latte input file and the TB + !! parametere. !!!WARNING!!! The parameters (cutoff, threshold, etc.) + !! need to be consistent with the gpmd input files. + !! + subroutine init_latte() + type(system_type) :: mysy +#ifdef USE_LATTE + !> Read the LATTE control file + call parse_control("latte.in") + call readtb() +#endif + end subroutine init_latte + + + !> Get the H and S based on the coordinates and symbols + !! \brief Constructing the latte Hamiltonian and Overlap matrices. + !! This is done with the information contained in the latte input + !! file and the TB parameters files. + !! \param myham0_bml Non-scf Hamiltonian bml format. + !! \param myover_bml Overlap matrix in bml format. + !! \param mycoords Coordinates of the system/sub-system. + !! coords(1,10): x-coordinate of atom 10. + !! \param mysymbols Symbol for every atom. + !! \param mylattice_vectors Lattice vectors. mylattice_vectors(1,:): First + !! lattice vector. + !! \param thresh Threshold value used to compute the matrices. + !! + subroutine get_hsmat_latte(myham0_bml,myover_bml,mycoords,mysymbols,& + mylattice_vectors,thresh) + implicit none + type(bml_matrix_t) :: myham0_bml,myover_bml + real(dp), allocatable :: mycoords(:,:) + character(2), allocatable :: mysymbols(:) + real(dp), allocatable :: mylattice_vectors(:,:) + real(dp) :: thresh + +#ifdef USE_LATTE + + nats = size(mycoords,dim=2) + if(allocated(cr)) deallocate(cr) + allocate(cr(3,nats)) + cr = mycoords + box = mylattice_vectors + if(allocated(atele)) deallocate(atele) + allocate(atele(nats)) + do i=1,nats + atele(i) = trim(adjustl(mysymbols(i))) + enddo + call readcr() + + !Get the dimension of the Hamiltonian + call gethdim() + IF (basistype .EQ. "NONORTHO")then + call allocatenono + endif + + !Get cutoff list. This might also be precomputed and passed + !for only the subsystem. + call gencutofflist() + + !!!WARNING + !Build integral mapp. This needs to be computed only once + !for the sull system and should be passed for the partial + !system through this routine. The integral map could become + !a memory problem if computed for the full sytem! + call build_integral_map() + + !Construct the Hamiltonian and Overlap + call genhonsite() + call genorbitallist() + call getmatindlist() + call allocatenebarrays() + + !!!WARNING + !This could be avoided since H and S for the part only need + !the tranlation vectors of the full system but not necessary the + !NL. At this point the NL was already applied. + call neblists(0) + + call bldnewhs() + + call bml_zero_matrix("dense",bml_element_real,latteprec,hdim,hdim,myham0_bml) + call bml_import_from_dense("dense", H, myham0_bml, thresh, hdim) + call bml_zero_matrix("dense",bml_element_real,latteprec,hdim,hdim,myover_bml) + call bml_import_from_dense("dense",smat , myover_bml, thresh, hdim) + + if(lt%verbose >= 1 .and. myRank == 1)then + call bml_print_matrix("ham_bml",myham0_bml,0,10,0,10) + call bml_print_matrix("over_bml",myover_bml,0,10,0,10) + endif + + call deallocatenono() + call deallocatenebarrays() + + !The following are variables which are deallocated + !in the deallocateall latte module which implies + !the use of many other modules not necessary for gpmd. + deallocate(f, fpp, ftot) + deallocate(deltaq, mycharge) + deallocate(elempointer) + deallocate(lcnshift) + deallocate(fpul, fscoul) + if(allocated(fsspin)) deallocate(fsspin) + if(allocated(respchi))deallocate(respchi) + deallocate(qlist) + deallocate(h,hdiag,h0,bo,h_onsite) + deallocate(cutoff_list) + deallocate(igl_map,orbital_list) + + + !stop + +#endif + end subroutine get_hsmat_latte + + + +end module gpmdcov_latte_mod + + diff --git a/examples/gpmdk/src/gpmdcov_lib.F90 b/examples/gpmdk/src/gpmdcov_lib.F90 new file mode 100644 index 00000000..ac2155b6 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_lib.F90 @@ -0,0 +1,138 @@ +!> Library interface +!! \brief This file is used to interface to python via iso_c_binding +!! library. +!! \param nats Number of total atoms in the system +!! \param nTypes Number of atom types +!! \param coords_in Coordinates of every atom in the system. +!! Allocation: +!! \verbatim coordinate(3,nats) \endverbatim +!! \param latticeVectors_in Flattened lattice vectors/box +!! Allocation: +!! \verbatim lattice_vector(3*3) \endverbatim +!! \verbatim v1 = lattice_vector(1:3) \endverbatim +!! \param atomTypes_in Atom type index for every atom +!! It gives the species index of a particulat atom. Indexing starts from 0! +!! Allocation: +!! \verbatim atomTypes(nats) \endverbatim +!! If we need the index of atom 30 then: +!! \verbatim atomTypes(30) \endverbatim +!! \param atomicNumbers_in Atomic number for every species. +!! A list with the atomic numbers for every species. +!! Allocation: +!! \verbatim atomicNumbers(nTypes) \endverbatim +!! \return forces_out Computed forces. Flattened 2D array. +!! \verbatim forces_out(2) \endverbatim : y component of the force for atom 1. +!! \param charges_out Vector of computed charges. +!! \param verb_in Verbosity level. +!! +function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomicNumbers_in,& + &charges_out,forces_out,dipole_out,bornch_out,verb_in) result(err) bind(c, name='gpmd_compute') + + use iso_c_binding, only: c_char, c_double, c_int, c_bool + use gpmdcov_vars + use gpmdcov_lib_mod + + implicit none + integer(c_int), intent(in), value :: nats + integer(c_int), intent(in), value :: nTypes + real(c_double), intent(inout) :: coords_in(3*nats) + real(c_double), intent(inout) :: forces_out(3*nats) + real(c_double), intent(inout) :: charges_out(nats) + real(c_double), intent(inout) :: dipole_out(3) + real(c_double), intent(inout) :: bornch_out(9*nats) + integer(c_int), intent(inout) :: atomTypes_in(nats) + integer(c_int) ,intent(inout) :: atomicNumbers_in(nTypes) + real(c_double) ,intent(inout) :: latticeVectors_in(9) + integer(c_int), intent(in), value :: verb_in + logical(c_bool) :: err + + real(dp), allocatable :: coords(:,:) + real(dp), allocatable :: forces(:,:), charges(:), dipole(:) + real(dp), allocatable :: bornch(:,:) + real(dp), allocatable :: latticeVectors(:,:) + integer, allocatable :: atomTypes(:), atomicNumbers(:) + integer :: k + integer :: verb + + err = .true. + allocate(coords(3,nats)) + allocate(atomTypes(nats)) + allocate(atomicNumbers(nTypes)) + allocate(latticeVectors(3,3)) + allocate(charges(nats)) + allocate(forces(3,nats)) + allocate(dipole(3)) + allocate(bornch(9,nats)) + + !Note that arrays appear in another order. We need to rearange + !the data. This is because of the column mayor (in python) vs. + !row mayor in fortran. + do k = 1, nats + coords(1,k) = coords_in((k-1)*3 + 1) + coords(2,k) = coords_in((k-1)*3 + 2) + coords(3,k) = coords_in((k-1)*3 + 3) + enddo + + latticeVectors(1,1) = latticeVectors_in(1) + latticeVectors(1,2) = latticeVectors_in(2) + latticeVectors(1,3) = latticeVectors_in(3) + + latticeVectors(2,1) = latticeVectors_in(4) + latticeVectors(2,2) = latticeVectors_in(5) + latticeVectors(2,3) = latticeVectors_in(6) + + latticeVectors(3,1) = latticeVectors_in(7) + latticeVectors(3,2) = latticeVectors_in(8) + latticeVectors(3,3) = latticeVectors_in(9) + + atomicNumbers = atomicNumbers_in + + do k = 1,nats !We correct for the indexing + atomTypes(k) = atomTypes_in(k) + 1 + enddo + + verb = verb_in + call gpmd_compute(coords,atomTypes,atomicNumbers,latticeVectors,& + &charges,forces,dipole,bornch,verb) + + err = .false. + + !We vectorize/flatten the forces to send back to python + do k = 1, nats + forces_out((k-1)*3 + 1) = forces(1,k) + forces_out((k-1)*3 + 2) = forces(2,k) + forces_out((k-1)*3 + 3) = forces(3,k) + enddo + + do k = 1, nats + bornch_out((k-1)*9 + 1) = bornch(1,k) + bornch_out((k-1)*9 + 2) = bornch(2,k) + bornch_out((k-1)*9 + 3) = bornch(3,k) + + bornch_out((k-1)*9 + 4) = bornch(4,k) + bornch_out((k-1)*9 + 5) = bornch(5,k) + bornch_out((k-1)*9 + 6) = bornch(6,k) + + bornch_out((k-1)*9 + 7) = bornch(7,k) + bornch_out((k-1)*9 + 8) = bornch(8,k) + bornch_out((k-1)*9 + 9) = bornch(9,k) + enddo + + charges_out(:) = charges(:) + dipole_out(:) = dipole(:) + + deallocate(coords) + deallocate(forces) + deallocate(charges) + deallocate(latticeVectors) + deallocate(atomTypes) + deallocate(atomicNumbers) + deallocate(dipole) + deallocate(bornch) + + return + +end function gpmd_fortran + + + diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 new file mode 100644 index 00000000..00ef1771 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -0,0 +1,219 @@ +module gpmdcov_lib_mod + + public :: gpmd_compute + +contains + +!> High-level program to perform GRAPH-BASED QMD. LIB MODULE +!! +!! \ingroup PROGRAMS +!! \brief High-level program to perform GRAPH-BASED QMD with a DFTB Hamiltonian +!! using a full parallel Graph-based approach. +!! \note To test this program with the \verbatim run_test \endverbatim script. +!! +!! \author C. F. A. Negre +!! (cnegre@lanl.gov) +!! +!! \author S. Mniszewski +!! (smn@lanl.gov) +!! +!! \author A. M. N. Niklasson +!! (amn@lanl.gov) +!! +!! \author M. E. Wall +!! (mewall@lanl.gov) +!! +!! Verbose levels: +!! >= 0- Only print tags. +!! >= 1- Print useful scalar physical data (e.g., Total Energy) +!! >= 2- Print single file output data (e.g., 0-SCF Charges) +!! >= 3- Write out trajectory data. +!! >= 4- Write out physically meaningful 1D arrays (e.g., Charges for the species) +!! >= 5- Write out physically meaningful 2D arrays (e.g., H matrix) +!! +!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& +! &forces_out,charges_out,verb_in) +!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& +! &forces_out,charges_out,verb_in) +!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& +! &charges_out,forces_out,verb_in) +subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + &charges_out,forces_out,dipole_out,bornch_out,verb_in) + + + ! Local modules + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_prepareMD_mod + use gpmdcov_mdloop_mod + use gpmdcov_init_mod + use gpmdcov_part_mod + use gpmdcov_assert_mod + use gpmdcov_mod + use gpmdcov_diagonalize_mod + use gpmdcov_kernel_mod + use gpmdcov_highorder_mod + use prg_ptable_mod + use prg_response_mod + + ! Latte modules +#ifdef USE_LATTE + use gpmdcov_latte_mod + use setuparray + use prg_system_mod + use latteparser + use constants_mod + use bml + use nonoarray + use NEBLISTARRAY +#endif + + implicit none + real(dp), allocatable, intent(in) :: coords_in(:,:), lattice_vectors_in(:,:) + real(dp), allocatable, intent(inout) :: charges_out(:) + real(dp), allocatable, intent(inout) :: forces_out(:,:) + real(dp), allocatable, intent(inout) :: bornch_out(:,:) + real(dp), allocatable, intent(inout) :: dipole_out(:) + integer, allocatable, intent(in) :: atomTypes_in(:) + integer, intent(in) :: verb_in + integer :: k + integer, allocatable, intent(in) :: atomic_numbers_in(:) + + real(dp),allocatable :: dipoleMoment(:),bornCharges(:,:) + real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) + real(dp) :: factor,dr,dfx,dfy,dfz + integer :: atomI + + !integer :: ipreMD +!!!!!!!!!!!!!!!!!!!!!!!! + !> Main program driver +!!!!!!!!!!!!!!!!!!!!!!!! + if(verb_in < 0)then + open(unit=6, file="/dev/null", form="formatted") + endif + + !> Initialize the program variables and parse input files. + sy%nats = size(atomTypes_in) + allocate(sy%symbol(sy%nats)) + allocate(sy%atomic_number(sy%nats)) + allocate(sy%coordinate(3,sy%nats)) + allocate(sy%mass(sy%nats)) + allocate(sy%lattice_vector(3,3)) + + sy%coordinate = coords_in + sy%lattice_vector = lattice_vectors_in + + !In this code symbols, mass, and atomic number are large + !arrays running through all the atoms, not only atom types. + !In gpmd, atomTypes are stored in sp%spindex + allocate(sy%spindex(sy%nats)) + write(*,*)"atom types",atomTypes_in + do k = 1,sy%nats + kk = atomTypes_in(k) + sy%spindex(k) = kk + sy%mass(k) = element_mass(atomic_numbers_in(kk)) + sy%symbol(k) = element_symbol(atomic_numbers_in(kk)) + sy%atomic_number(k) = atomic_numbers_in(kk) + enddo + + sy%nsp = size(atomic_numbers_in) + allocate(sy%splist(sy%nsp)) + allocate(sy%spatnum(sy%nsp)) + allocate(sy%spmass(sy%nsp)) + + do k = 1,sy%nsp + sy%splist(k) = element_symbol(atomic_numbers_in(k)) + sy%spatnum(k) = atomic_numbers_in(k) + sy%spmass(k) = element_mass(atomic_numbers_in(k)) + enddo + write(*,*)"spindex",sy%spindex,"splist",sy%splist + + !deallocate(coords_in, attice_vectors_in) + !deallocate(atomTypes_in) + + write(*,*)"Bef init" + call gpmdcov_Init(.true.) + + lt%stopat = "gpmdcov_Energ" + lt%verbose = verb_in + + +#ifdef USE_LATTE + call init_latte() +#endif + write(*,*)"Bef asset" + call gpmdcov_assert_input(myRank) + + !We give a first guess of the Fermi level. + Ef = lt%efermi + + !The inverse of the electronic temperature. + beta = 1.0_dp/lt%kbt + + !> Initial partition of the system based on the covalency graph. + call gpmdcov_Part(1) + if(lt%stopAt == "gpmdcov_Part") stop + + !> Initialize partitions. + call gpmdcov_InitParts() + if(lt%stopAt == "gpmdcov_InitParts") stop + !> Comput first charges. + if(lt%method == "DiagEfFull") eig = .false. + if(.not. eig) then + call gpmdcov_Diagonalize_H0() + if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then + call gpmdcov_muFromParts() + endif + endif + call gpmdcov_FirstCharges(eig) + if(lt%stopAt == "gpmdcov_FirstCharges") return + + + !> First SCF loop up to maxscf. + if(eig)then + call gpmdcov_DM_Min(lt%maxscf,sy%net_charge,.true.) + else + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + endif + + if(lt%stopAt == "gpmdcov_DM_Min") return + + !> First calculation of energies and forces. + call gpmdcov_EnergAndForces(sy%net_charge) + + charges_out(:) = sy%net_charge(:) + forces_out(:,:) = sy%force(:,:) + + if(allocated(dipoleMoment)) deallocate(dipoleMoment) + allocate(dipoleMoment(3)) + factor = 1.0_dp + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + dipole_out(:) = dipoleMoment(:) + write(*,*)"Dipole Moment=",dipoleMoment + + !call gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) + call gpmdcov_get_born_charges_v2(dipoleMoment,bornCharges,tb%norbi) + bornch_out(:,:) = bornCharges(:,:) + write(*,*)"Born F",bornCharges(:,1) + + if(lt%stopAt == "gpmdcov_Energ") then + lib_init = .true. + call gpmdcov_Finalize() + return + endif + + !> Setup the Molecular Dynamics (MD) calculation. + call gpmdcov_PrepareMD(gpmdt%temp0) + if(lt%stopAt == "gpmdcov_PrepareMD") return + + !> Perform the MD simulation. + call gpmdcov_MDloop() + if(lt%stopAt == "gpmdcov_MDloop")then + call gpmdcov_Finalize() + return + endif + + end subroutine gpmd_compute + +end module gpmdcov_lib_mod diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 new file mode 100644 index 00000000..aede9769 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -0,0 +1,532 @@ +module gpmdcov_MDloop_mod + +contains + + !> Main MD loop + !! This routine performs the MD loops up to "ls%mdsteps" + !! + subroutine gpmdcov_MDloop() + + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_part_mod + use gpmdcov_writeout_mod + use gpmdcov_kernel_mod + use gpmdcov_neighbor_mod + use gpmdcov_langevin_mod + use gpmdcov_preparemd_mod + + real(dp) :: mls_md, mls_md1, mls_md2, resnorm + real(dp), allocatable :: kernelTimesRes(:), n1(:) + real(dp), allocatable :: KK0Res(:) + type(system_type) :: syaux + integer, allocatable :: seedin(:) + integer :: myseed,ssize + real, allocatable :: langevin_rands(:,:,:) + real(dp), allocatable :: DV(:,:) + integer :: maxv_atom_axis(2) + integer :: nanv_atom_axis(2) + real(dp) :: virial(3,3) + real(dp) :: ke_tensor(3,3) + real(dp) :: pressure_tensor(3,3) + integer :: total_steps + + ! Prepare for Langevin dynamics if needed + if(gpmdt%langevin)then + + !Initialize random number generator + myseed = 12345 + call random_seed() + call random_seed(size=ssize) + allocate(seedin(ssize)) + seedin = myseed + call random_seed(PUT=seedin) + !Allocate array of random numbers for Langevin integration + allocate(langevin_rands(sy%nats,3,2)) + !Allocate delta_V arrays + allocate(DV(3,sy%nats)) + endif + + call gpmdcov_msI("gpmdcov_MDloop","In gpmdcov_MDloop ...",lt%verbose,myRank) + savets = lt%timestep + !do mdstep = -1,lt%mdsteps + if(.not.gpmdt%restartfromdump.and.gpmdt%minimization_steps.ne.0)then + sy%velocity = 0.0_dp + endif + ! Compute box volume + call gpmdcov_get_vol(sy%lattice_vector,sy%volr) + + total_steps = lt%mdsteps + gpmdt%minimization_steps + + do mdstep = 1,total_steps + ! if(mdstep < 0)then + ! savets = lt%timestep + ! lt%timestep = 0 + ! else + ! lt%timestep = savets + ! endif + + + mls_md = mls() + mls_md1 = mls() + + if(myRank == 1)then + write(*,*)"" + write(*,*)" #######################" + if(mdstep.le.gpmdt%minimization_steps)then + write(*,*)" Min Step =",mdstep + else + write(*,*)" MDStep =",mdstep-gpmdt%minimization_steps + endif + write(*,*)" #######################" + write(*,*)"" + endif + + maxv_atom_axis = MAXLOC(ABS(sy%velocity)) + call gpmdcov_msI("gpmdcov_MDloop","Maximum Velocity "//to_string(MAXVAL(ABS(sy%velocity)))//" & + &for (atom,axis) = ("//to_string(maxv_atom_axis(2))//","//to_string(maxv_atom_axis(1))//")",lt%verbose,myRank) + + !> Get Kinetic energy + EKIN = 0.0_dp + do i=1,sy%nats + EKIN = EKIN + & + & sy%mass(i)*(sy%velocity(1,i)**2+sy%velocity(2,i)**2+sy%velocity(3,i)**2) + enddo + EKIN = 0.5_dp*MVV2KE*EKIN + + !! Statistical temperature in Kelvin + Temp = (2.0_dp/3.0_dp)*KE2T*EKIN/real(sy%nats,dp); + !! Total Energy in eV + Energy = EKIN + EPOT; + !! Time in fs + Time = mdstep*lt%timestep; + + !! Statistical pressure + do i = 1,3 + do j = 1,3 + ke_tensor(i,j) = MVV2KE*sum(sy%mass(:)*sy%velocity(i,:)*sy%velocity(j,:)) + virial(i,j) = MVV2KE*F2V*sum(sy%coordinate(i,:)*sy%force(j,:)) + enddo + enddo + + pressure_tensor = EVOVERV2P*(ke_tensor + virial)/sy%volr + + if(myRank == 1)then + write(*,*)"Time [fs] = ",Time + write(*,*)"Energy Kinetic [eV] = ",EKIN + write(*,*)"Energy Potential [eV] = ",EPOT + write(*,*)"Energy Total [eV] = ",Energy + write(*,*)"Temperature [K] = ",Temp + write(*,*)"Pressure [bar] = ",pressure_tensor(1,1)+pressure_tensor(2,2)+pressure_tensor(3,3) + endif + + call gpmdcov_msI("gpmdcov_MDloop","Time for Preliminars "//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + mls_md1 = mls() + + if(.not.gpmdt%langevin)then + !> First 1/2 of Leapfrog step + call gpmdcov_msMem("gpmdcov_mdloop", "Before halfVerlet",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) call prg_timer_start(dyn_timer,"Half Verlet") + call halfVerlet(sy%mass,sy%force,lt%timestep,sy%velocity(1,:),sy%velocity(2,:),sy%velocity(3,:)) + if(lt%verbose >= 1) call prg_timer_stop(dyn_timer,1) + call gpmdcov_msMem("gpmdcov_mdloop", "After halfVerlet",lt%verbose,myRank) + + if(myRank == 1 .and. lt%verbose.GE.5)then + do i = 1,sy%nats + write(*,*)i,sy%velocity(1,i),sy%velocity(2,i),sy%velocity(3,i) + enddo + endif + !> Update positions + call gpmdcov_msMem("gpmdcov_mdloop", "Before updatecoords",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) call prg_timer_start(dyn_timer,"Update positions") + call updatecoords(origin,sy%lattice_vector,lt%timestep,sy%velocity(1,:),sy%velocity(2,:),sy%velocity(3,:),sy%coordinate) + if(myRank == 1 .and. lt%verbose >= 1) call prg_timer_stop(dyn_timer,1) + call gpmdcov_msMem("gpmdcov_mdloop", "After updatecoords",lt%verbose,myRank) +#ifdef DO_MPI + if (numRanks .gt. 1) then !THIS IS VERY IMPORTANT + call prg_sumRealReduceN(sy%coordinate(1,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(2,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(3,:), sy%nats) + + call prg_sumRealReduceN(sy%velocity(1,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(2,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(3,:), sy%nats) + + sy%coordinate = sy%coordinate/real(numRanks,dp) + sy%velocity = sy%velocity/real(numRanks,dp) + endif +#endif + + call gpmdcov_msI("gpmdcov_MDloop","Time for halfVerlet, Update coords, and sumReduce "& + &//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + mls_md1 = mls() + endif + + !call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) + !origin = 0.0_dp + + if(gpmdt%langevin.and.gpmdt%langevin_method.eq."Siva")then + call random_number(langevin_rands) + call gpmdcov_uniform_to_normal(langevin_rands) + !if (myRank.eq.0)then + ! call random_number(langevin_rands) + ! call uniform_to_normal(langevin_rands) + !endif + !if (numranks.gt.1)then + ! call prg_bcastRealParallel(langevin_rands,size(langevin_rands),0) + !endif + call gpmdcov_msI("gpmdcov_MDloop","Langevin Siva integration with (gamma,temp) = ("& + &//to_string(gpmdt%langevin_gamma)//","//to_string(gpmdt%temp0)//")",lt%verbose,myRank) + call LangevinDVSivaOne(sy%mass,sy%force,lt%timestep,sy%velocity,gpmdt%langevin_gamma,gpmdt%temp0,langevin_rands) + call updatecoords(origin,sy%lattice_vector,lt%timestep,sy%velocity(1,:),sy%velocity(2,:),sy%velocity(3,:),sy%coordinate) +#ifdef DO_MPI + if (numRanks .gt. 1) then !THIS IS VERY IMPORTANT + call prg_sumRealReduceN(sy%coordinate(1,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(2,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(3,:), sy%nats) + + call prg_sumRealReduceN(sy%velocity(1,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(2,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(3,:), sy%nats) + + sy%coordinate = sy%coordinate/real(numRanks,dp) + sy%velocity = sy%velocity/real(numRanks,dp) + endif +#endif + endif + + if(mdstep >= 1)then + if(lt%doKernel)then + + if(kernel%xlbolevel1)then + call gpmdcov_msI("gpmdcov_MDloop","Doing XLBO level 1",lt%verbose,myRank) + if(kernel%kernelType == "ByParts")then + !propagate n + allocate(kernelTimesRes(sy%nats)) + if(mdstep.le.1)then + n = sy%net_charge + call gpmdcov_applyKernel(sy%net_charge,n,syprtk,KK0Res) + call prg_xlbo_nint_kernelTimesRes(sy%net_charge,n,n_0,& + &n_1,n_2,n_3,n_4,n_5,mdstep,KK0Res,xl) + endif + if(mdstep > 1 .and. kernel%rankNUpdate > 0 .and. & + & mod(mdstep,kernel%updateEach) == 0)then + call gpmdcov_msI("gpmdcov_MDloop","Integrating n ...",lt%verbose,myRank) + + !call gpmdcov_applyKernel(sy%net_charge,n,syprtk,KK0Res) + call prg_xlbo_nint_kernelTimesRes(sy%net_charge,n,n_0,& + &n_1,n_2,n_3,n_4,n_5,mdstep,KK0Res,xl) + !Use n > H > to get q_min + ! call gpmdcov_DM_Min_Eig(1,sy%net_charge,.false.) + !Compute KK0Res + ! call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) + !Use KK0Res to update n to and n1 + ! sy%net_charge = n - KK0Res + !Compute H again and get q_min from n1 later in the code + endif + deallocate(kernelTimesRes) + else + STOP "XLBOLevel1 not implemented for other than kernelType= ByParts" + endif + + else + if(kernel%kernelType == "ByParts")then + allocate(kernelTimesRes(sy%nats)) + if(mdstep.le.1)then + n = sy%net_charge + endif + if(mdstep > 1 .and. kernel%rankNUpdate > 0 .and. & + & mod(mdstep,kernel%updateEach) == 0)then + mls_md2 = mls() + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_rankN_update_byParts",lt%verbose,myRank) + call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_rankN_update_byParts",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_rankN_update_byParts "& + &//to_string(mls() - mls_md2)//" ms",lt%verbose,myRank) + if(gpmdt%trackreactivity)then + syaux%nats = sy%nats + allocate(syaux%symbol(sy%nats)); syaux%symbol = sy%symbol + allocate(syaux%coordinate(3,sy%nats)); syaux%coordinate = sy%coordinate + allocate(syaux%net_charge(sy%nats)); syaux%net_charge = KK0Res*100.0_dp + if(myRank == 1)then + call prg_write_trajectory(syaux,mdstep,gpmdt%writetreach,lt%timestep,& + &adjustl(trim(lt%jobname))//"_reactivity","xyz") + endif + call prg_destroy_system(syaux) + endif + + else + mls_md2 = mls() + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_applyKernel",lt%verbose,myRank) + call gpmdcov_applyKernel(sy%net_charge,n,syprtk,KK0Res) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_applyKernel",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_applyKernel "& + &//to_string(mls() - mls_md2)//" ms",lt%verbose,myRank) + endif + call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_nint_kernelTimesRes",lt%verbose,myRank) + call prg_xlbo_nint_kernelTimesRes(sy%net_charge,n,n_0,& + &n_1,n_2,n_3,n_4,n_5,mdstep,KK0Res,xl) + call gpmdcov_msMem("gpmdcov_mdloop", "After prg_xlbo_nint_kernelTimesRes",lt%verbose,myRank) + deallocate(kernelTimesRes) + else + call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_nint_kernel",lt%verbose,myRank) + call prg_xlbo_nint_kernel(sy%net_charge,n,n_0,n_1,n_2,n_3,n_4,n_5,mdstep,Ker,xl) + call gpmdcov_msMem("gpmdcov_mdloop", "After prg_xlbo_nint_kernel",lt%verbose,myRank) + endif + endif + else + call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_nint",lt%verbose,myRank) + call prg_xlbo_nint(sy%net_charge,n,n_0,n_1,n_2,n_3,n_4,n_5,mdstep,xl) + call gpmdcov_msMem("gpmdcov_mdloop", "After prg_xlbo_nint",lt%verbose,myRank) + endif + else + n = sy%net_charge + endif + call gpmdcov_msI("gpmdcov_MDloop","Time for prg_xlbo_nint "//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + !> Update neighbor list (Actialized every nlisteach times steps) + mls_md1 = mls() + if(mod(mdstep,lt%nlisteach) == 0 .or. mdstep == 0 .or. mdstep == 1)then + call gpmdcov_msMem("gpmdcov_mdloop", "Before build_nlist_int",lt%verbose,myRank) + call gpmdcov_destroy_nlist(nl,lt%verbose) + !call destroy_nlist(nl) + if(nlistSparse)then + call gpmdcov_build_nlist_sparse_v2(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) + else + call gpmdcov_build_nlist_full(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) + endif + !LBox(1) = sy%lattice_vector(1,1) + !LBox(2) = sy%lattice_vector(2,2) + !LBox(3) = sy%lattice_vector(3,3) + + !call gpmdcov_nearestneighborlist(nl%nrnnlist,nl%nndist,nl%nnRx,nl%nnRy,nl%nnRz,nl%nnType, & + ! &sy%coordinate(1,:),sy%coordinate(2,:),sy%coordinate(2,:),LBox,coulcut,sy%nats,200) + + !call build_nlist_int(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose) + call gpmdcov_msMem("gpmdcov_mdloop", "After build_nlist_int",lt%verbose,myRank) + endif + call gpmdcov_msI("gpmdcov_MDloop","Time for build_nlist_int "& + &//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + !stop + !> Repartition. + ! This builds the new graph. + mls_md1 = mls() + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_Part",lt%verbose,myRank) + call gpmdcov_Part(2) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_Part",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_Part & + &"//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + !> Reprg_initialize parts. + mls_i = mls() + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_InitParts",lt%verbose,myRank) + call gpmdcov_InitParts() + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_InitParts",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_InitParts & + &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + mls_md1 = mls() + resnorm = 0.0_dp + + if((mdstep >= 2) .and. (.not. kernel%xlbolevel1)) resnorm = norm2(sy%net_charge - n)/sqrt(dble(sy%nats)) + + Nr_SCF_It = xl%maxscfiter; + !> Use SCF the first MD steps + if(mdstep < xl%minit)then + Nr_SCF_It = xl%maxscfInitIter + else + Nr_SCF_It = xl%maxscfiter + endif + + !> SCF loop + !if(newPart .and. (.not.lt%dokernel)) then + ! Nr_SCF_It = xl%maxscfInitIter + ! newPart = .false. + !endif + + + if(Nr_SCF_It .gt. 0)then + if(eig)then + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min",lt%verbose,myRank) + call gpmdcov_dm_min(Nr_SCF_It,n,.true.) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min",lt%verbose,myRank) + else + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min_Eig",lt%verbose,myRank) + call gpmdcov_dm_min_Eig(Nr_SCF_It,n,.true.,.false.) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min_Eig",lt%verbose,myRank) + endif + endif + + sy%net_charge = n + + if(eig)then + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min",lt%verbose,myRank) + call gpmdcov_DM_Min(1,sy%net_charge,.false.) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min",lt%verbose,myRank) + else + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min_Eig",lt%verbose,myRank) + call gpmdcov_DM_Min_Eig(1,sy%net_charge,.false.,.false.) + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min_Eig",lt%verbose,myRank) + endif + + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_DM_Min_1 & + &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + if(kernel%xlbolevel1)then + allocate(n1(sy%nats)) + if(mdstep > 1)then + !sy%net_charge = n + !Compute KK0Res + call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) + !Use KK0Res to update n to and n1 + n1 = n - KK0Res + sy%net_charge = n1 + call gpmdcov_DM_Min_Eig(1,sy%net_charge,.false.,.false.) + !Compute H again and get q_min from n1 later in the code + resnorm = norm2(sy%net_charge - n1)/sqrt(dble(sy%nats)) + endif + endif + + + mls_md1 = mls() + call gpmdcov_msI("gpmdcov_MDloop","ResNorm = "//to_string(resnorm),lt%verbose,myRank) + if(myRank == 1)then + if(mdstep.le.gpmdt%minimization_steps)then + write(*,'(A35,I15,A1,F18.5,A1,ES12.5,A1,ES12.5,A1,ES12.5)')"Minstep, Energy, Egap, Resnorm, Temp", & + &mdstep," ", Energy," ", egap_glob," ", resnorm," ", Temp + else + write(*,'(A35,I15,A1,F18.5,A1,ES12.5,A1,ES12.5,A1,ES12.5)')"Mdstep, Energy, Egap, Resnorm, Temp", & + &mdstep-gpmdt%minimization_steps," ", Energy," ", egap_glob," ", resnorm," ", Temp + endif + !write(*,*)"Step, Energy, EGap, Resnorm", mdstep, Energy, egap_glob, resnorm + endif + + call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_EnergAndForces",lt%verbose,myRank) + if(kernel%xlbolevel1)then + if(mdstep <= 1) n1 = n + call gpmdcov_EnergAndForces(n1) + deallocate(n1) + else + call gpmdcov_EnergAndForces(n) + endif + call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_EnergAndForces",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_EnergAndForces & + &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + mls_md1 = mls() + !> Adjust forces for the linearized XLBOMD functional + call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_fcoulupdate",lt%verbose,myRank) + call prg_xlbo_fcoulupdate(Coul_Forces,sy%net_charge,n) + call gpmdcov_msMem("gpmdcov_mdloop", "After prg_xlbo_fcoulupdate",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for prg_xlbo_fcoulupdate & + &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + mls_md1 = mls() + + !> Total XLBOMD force + ! sy%force = SKForce + PairForces + FPUL + Coul_Forces + + ! FSCOUL; + sy%force = collectedforce + PairForces + Coul_Forces + + !> Integrate second 1/2 of leapfrog step + if(gpmdt%dovelresc .eqv. .true.)then + call gpmdcov_msI("gpmdcov_MDloop","Doing Velocity Rescale",lt%verbose,myRank) + if(gpmdt%temp0 .ne. 0.0_dp)then + sy%velocity = sqrt(gpmdt%temp0/Temp)*sy%velocity + else + sy%velocity = gpmdt%velresc_fact*sy%velocity + endif + endif + + + if(.not.gpmdt%restartfromdump)then + if(mdstep.lt.gpmdt%minimization_steps)then + call gpmdcov_msI("gpmdcov_MDloop","Zeroing velocities during minimization",lt%verbose,myRank) + sy%velocity = 0.0_dp + elseif(mdstep.eq.gpmdt%minimization_steps)then + if(.not.gpmdt%langevin.and.gpmdt%temp0.gt.1.0E-10)then + call gpmdcov_addVelocity(gpmdt%temp0,sy%velocity,sy%mass) + else + sy%velocity = 0.0_dp + endif + endif + endif + + call gpmdcov_msMem("gpmdcov_mdloop", "Before halfVerlet",lt%verbose,myRank) + if(gpmdt%langevin.and.gpmdt%langevin_method.eq."Siva")then + call random_number(langevin_rands) + call gpmdcov_uniform_to_normal(langevin_rands) +! if (myRank.eq.0)then +! call random_number(langevin_rands) +! call uniform_to_normal(langevin_rands) +! endif +! if (numranks.gt.1)then +! call prg_bcastRealParallel(langevin_rands,size(langevin_rands),0) +! endif + call LangevinDVSivaTwo(sy%mass,sy%force,lt%timestep,sy%velocity,gpmdt%langevin_gamma,gpmdt%temp0,langevin_rands) + endif + + if(gpmdt%langevin.and.gpmdt%langevin_method.eq."Goga")then + call random_number(langevin_rands) + call gpmdcov_uniform_to_normal(langevin_rands) + call gpmdcov_msI("gpmdcov_MDloop","Langevin integration with (gamma,temp) = ("& + &//to_string(gpmdt%langevin_gamma)//","//to_string(gpmdt%temp0)//")",lt%verbose,myRank) + !First update the velocities using the force field and compute DV + call LangevinDVGoga(sy%mass,sy%force,lt%timestep,sy%velocity,DV,gpmdt%langevin_gamma,gpmdt%temp0,langevin_rands) + !Next update the coordinates using the updated velocities and DV + call LangevinDXGoga(origin,sy%lattice_vector,lt%timestep,sy%velocity(1,:),sy%velocity(2,:),sy%velocity(3,:),& + &DV(1,:),DV(2,:),DV(3,:),sy%coordinate) + !Finally update the velocities using DV + sy%velocity(:,:) = sy%velocity(:,:) + DV(:,:) + call gpmdcov_msMem("gpmdcov_mdloop", "After Langevin",lt%verbose,myRank) +#ifdef DO_MPI + if (numRanks .gt. 1) then !THIS IS VERY IMPORTANT + call prg_sumRealReduceN(sy%coordinate(1,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(2,:), sy%nats) + call prg_sumRealReduceN(sy%coordinate(3,:), sy%nats) + + call prg_sumRealReduceN(sy%velocity(1,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(2,:), sy%nats) + call prg_sumRealReduceN(sy%velocity(3,:), sy%nats) + + sy%coordinate = sy%coordinate/real(numRanks,dp) + sy%velocity = sy%velocity/real(numRanks,dp) + endif +#endif + + else + call halfVerlet(sy%mass,sy%force,lt%timestep,sy%velocity(1,:),sy%velocity(2,:),sy%velocity(3,:)) + call gpmdcov_msMem("gpmdcov_mdloop", "After halfVerlet",lt%verbose,myRank) + endif + + if(gpmdt%writetraj .and. myRank == 1 .and. mdstep.ge.gpmdt%minimization_steps)then + if(gpmdt%traj_format .eq. "XYZ")then + call prg_write_trajectory(sy,mdstep-gpmdt%minimization_steps,gpmdt%writetreach,& + <%timestep,adjustl(trim(lt%jobname))//"_trajectory","xyz") + else + call prg_write_trajectory(sy,mdstep-gpmdt%minimization_steps,gpmdt%writetreach,& + <%timestep,adjustl(trim(lt%jobname))//"_trajectory","pdb") + endif + endif + if(gpmdt%dumpeach .gt. 0)then + if(mod(mdstep,gpmdt%dumpeach) .eq. 0)then + call gpmdcov_dump() + endif + endif + call gpmdcov_msI("gpmdcov_MDloop","Time for rest, inc halfVerlet & + &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + + call gpmdcov_msI("gpmdcov_MDloop","Time for MD iter & + &"//to_string(mls() - mls_md)//" ms",lt%verbose,myRank) + + ! Save MD state each 120 steps + if(gpmdt%dumpeach .gt. 0)then + if(mod(mdstep,gpmdt%dumpeach) == 0)call gpmdcov_dump() + endif + enddo + ! End of MD loop. + + end subroutine gpmdcov_MDloop + +end module gpmdcov_MDloop_mod diff --git a/examples/gpmdk/src/gpmdcov_mod.F90 b/examples/gpmdk/src/gpmdcov_mod.F90 new file mode 100644 index 00000000..784aa21e --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_mod.F90 @@ -0,0 +1,409 @@ +!> Auxiliary modules. +!! \brief This module will be used to have auxiliary routines. +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_mod + + + private + + integer, parameter :: dp = kind(1.0d0) + logical :: err + + public :: gpmdcov_musearch, gpmdcov_musearch_bisec + public :: gpmdcov_muDyn, gpmdcov_muFromParts, algo + public :: gpmdcov_fermifunction + +contains + + subroutine gpmdcov_fermifunction(beta,energies,mu,fermi) + real(dp), intent(in) :: beta + real(dp), intent(in) :: energies(:) + real(dp), intent(in) :: mu + real(dp), intent(out) :: fermi(:) + real(dp) :: biggest + real(dp), allocatable :: exparg(:) + + if(.not.allocated(exparg))then + allocate(exparg(size(energies))) + endif + + biggest = log(HUGE(biggest))-1.0_dp + exparg = beta*(energies(:)-mu) + exparg = merge(exparg,biggest,exparg.le.biggest) + fermi = 1.0_dp/(exp(exparg) + 1.0_dp) + + end subroutine gpmdcov_fermifunction + + subroutine gpmdcov_muDyn(nguess,Nr_SCF) + use gpmdcov_vars + use gpmdcov_writeout_mod + real(dp), allocatable :: nguess(:) + integer, intent(in) :: Nr_SCF + + !Actualize the Fermi level dynamically + + if(.not.allocated(acceprat))then + allocate(acceprat(2)) + acceprat = 0 + Efstep = 0.1_dp + endif + + acceprat(2) = acceprat(1) + acceprat(1) = sign(1.0_dp,tch) + + if(acceprat(2)*acceprat(1) < 0)then + Efstep = Efstep*0.8_dp + else + Efstep = Efstep*1.01_dp + endif + + if(Nr_SCF.gt.10)then + if(iscf.gt.10)Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) + else + Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) + endif + + !Normalize charges to tch + nguess(:) = nguess(:) - tch/real(sy%nats) + + call gpmdcov_msI("gpmdcov_muDyn","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) + + end subroutine gpmdcov_muDyn + + subroutine gpmdcov_muFromParts() + + use gpmdcov_vars + use gpmdcov_writeout_mod + integer :: mycount + real(dp) :: mls_mu + + call gpmdcov_msI("gpmdcov_getmu","In gpmdcov_getmu ...",lt%verbose,myRank) + + norbsInRank = sum(norbsInEachCHAtRank) + + if(allocated(evalsInRank))then + deallocate(evalsInRank) + deallocate(dvalsInRank) + endif + + allocate(evalsInRank(norbsInRank)) + allocate(dvalsInRank(norbsInRank)) + + evalsInRank = 0.0_dp + dvalsInRank = 0.0_dp + shift = 0 + mycount = 0 + +#ifdef DO_MPI + do iptt=1,partsInEachRank(myRank) + ipt= reshuffle(iptt,myRank) +#else + do iptt = 1,gpat%TotalParts + ipt = iptt +#endif + do i = 1, norbsInEachCHAtRank(iptt) + evalsInRank(i+shift) = syprt(ipt)%estr%evals(i) + dvalsInRank(i+shift) = syprt(ipt)%estr%dvals(i) + mycount = mycount + 1 + enddo + shift = shift + norbsInEachCHAtRank(iptt) + enddo + +! write(*,*)"norbsInRank",norbsInRank +! write(*,*)"evalsInEachCHAtRank",norbsInEachCHAtRank + + if(.not.allocated(norbsInEachCH)) allocate(norbsInEachCH(gpat%TotalParts)) + norbsInEachCH = 0 + nRanks = getNRanks() + if(.not.allocated(npartsVect))then + allocate(npartsVect(nRanks)) + allocate(displ(nRanks)) + allocate(norbsInEachRank(nRanks)) + endif + + do i = 1,nRanks + npartsVect(i) = partsInEachRank(i) + if(i == 1)then + shift = 0 + else + shift = shift + partsInEachRank(i-1) + endif + displ(i) = shift + enddo + +#ifdef DO_MPI + call allGatherVIntParallel(norbsInEachCHAtRank, partsInEachRank(myRank),norbsInEachCH ,npartsVect, displ) +#else + norbsInEachCH = norbsInEachCHAtRank +#endif + + kk = 0 + + norbsInEachRank = 0.0_dp + do i = 1,nRanks + do j = 1,partsInEachRank(i) + kk = kk + 1 + norbsInEachRank(i) = norbsInEachRank(i) + norbsInEachCH(kk) + enddo + if(i == 1)then + shift = 0 + else + shift = shift + norbsInEachRank(i-1) + endif + displ(i) = shift + enddo + totalNorbs = sum(norbsInEachRank) + + if(allocated(evalsAll))then + deallocate(evalsAll) + deallocate(dvalsAll) + endif + allocate(evalsAll(totalNorbs)) + allocate(dvalsAll(totalNorbs)) + evalsAll = 0.0_dp + dvalsAll = 0.0_dp + +#ifdef DO_MPI + call allGatherVRealParallel(evalsInRank, norbsInRank, evalsAll ,norbsInEachRank, displ) + call allGatherVRealParallel(dvalsInRank, norbsInRank, dvalsAll ,norbsInEachRank, displ) +#else + evalsAll = evalsInRank + dvalsAll = dvalsInRank +#endif + + + nocc = bndfilTotal*real(sy%estr%norbs,dp) + + err = .false. + + mls_mu = mls() + call gpmdcov_musearch(evalsAll,dvalsAll,beta,nocc,10000,10d-10,& + &Ef,HOMO,LUMO,err,myRank,egap_glob,lt%verbose) + if(err .eqv. .true.)then + call gpmdcov_musearch_bisec(evalsAll,dvalsAll,beta,nocc,10d-10,Ef,myRank,egap_glob,lt%verbose) + endif + + call gpmdcov_msI("gpmdcov_muFromParts","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_muFromParts","Time for mu search="//to_string(mls() - mls_mu),lt%verbose,myRank) + + end subroutine gpmdcov_muFromParts + + !> Perform musearch. + !! \param evals Eigenvalues of the system. + !! \param dvals Partial traces. + !! \prarm beta Electronic temperature. + !! \param maxMuIter Maximum numner of iterations. + !! \param occTol Occupation tolerance. + !! \param mu Chemical potential. + !! + subroutine gpmdcov_musearch(evals, dvals, beta, nocc, maxMuIter, occTol, mu, & + & HOMO, LUMO, err, rank, egap, verbose) + use gpmdcov_writeout_mod + implicit none + integer :: i, j, norbs + integer, intent(in) :: maxMuIter, rank + real(dp) :: den, occErr, mu0, occ, muMax,muMin + real(dp), allocatable :: fvals(:) + real(dp), intent(in) :: evals(:), dvals(:) + real(dp), intent(in) :: occTol, beta, nocc + real(dp), intent(inout) :: egap + real(dp), intent(inout) :: mu, HOMO, LUMO + logical, intent(inout) :: err + integer, optional, intent(in) :: verbose + + mu0 = 0.5d0*(evals(int(floor(Nocc))) + evals(int(floor(Nocc))+1)) + muMin = minval(evals) + muMax = maxval(evals) + if (present(verbose)) then + call gpmdcov_msI("gpmdcov_musearch","In gpmdcov_musearch with muMin,muMax,mu0 & + &= "//to_string(muMin)//","//to_string(muMax)//","//to_string(mu0),verbose,rank) + endif + norbs = size(evals, dim = 1) + + if(.not.allocated(fvals))then + allocate(fvals(norbs)) + endif + + ! if(mu > (muMin - muMax)/2.0_dp)then + ! mu = (muMin - muMax)/2.0_dp + ! else + ! mu0 = mu !(muMin - muMax)/2.0_dp + ! endif + + call gpmdcov_fermifunction(beta,evals,mu0,fvals) + + !do j = 1, norbs + ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) + !end do + + occ = 0.0_dp + do j = 1, norbs + occ = occ + fvals(j)*dvals(j) + end do + + occErr = abs(occ-nocc) + + do i = 1, maxMuIter + + call gpmdcov_fermifunction(beta,evals,mu0,fvals) + !do j = 1, norbs + ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) + !end do + + occ = 0.0_dp + den = 0.0_dp + do j = 1, norbs + occ = occ + fvals(j)*dvals(j) + den = den + beta*fvals(j)*(1.0_dp-fvals(j))*dvals(j) + end do + + occErr = abs(occ - nocc) + if (occErr < occTol) then + mu = mu0 + exit + elseif (den > 1.0d-12) then + mu0 = mu0 + (nocc - occ)/den + mu = mu0 + endif + + if(mu > muMax .or. mu < muMin)then + err = .true. + mu = 0.0_dp + return + endif + + if (i .eq. maxMuIter) then + write(*,*) "Max mu iters reached for NR. Trying bisection ..." + err = .true. + mu = 0.0_dp + return + else + if(occErr .lt. occTol) exit + end if + + end do + + deallocate(fvals) + + HOMO = muMin + LUMO = muMax + do i = 1,norbs + if(evals(i) .lt. mu)then + HOMO = max(evals(i),HOMO) + elseif(evals(i) .gt. mu)then + LUMO = min(evals(i),LUMO) + endif + enddo + + egap = LUMO-HOMO + + end subroutine gpmdcov_musearch + + + !> Routine to compute the Fermi level given a set of eigenvalues and a temperature. + !! It applies the Bisection method over the function: + !! \f$ g(\mu) = \sum_k 2 f(\epsilon_k - \mu) - N = 0 \f$ + !! Where \f$ f(\epsilon_k - \mu) = \frac{1}{1+\exp{(\epsilon_k - \mu)/(k_bT)}}\f$. + !! \param evals Eigenvalues of the system (\f$ \{ \epsilon_k \} \f$). + !! \param kbt Temperature times the Boltzmann's constant (\f$ k_bT \f$). + !! \param bndfil Filing factor (\f$ N_{el}/(2*N_{orbs})\f$). + !! \param tol Tolerance for the bisection method. + !! \param Ef Fermi level (\f$ \mu \f$). + !! + subroutine gpmdcov_musearch_bisec(evals,dvals,beta,noc,tol,mu,rank,egap,verbose) + use gpmdcov_writeout_mod + integer :: i, m + integer :: norb + integer, intent(in) :: rank + integer, optional, intent(in) :: verbose + real(dp) :: Ft1, Ft2, Prod, egap + real(dp) :: step, tol, fermi, mumax, muMin, HOMO, LUMO + real(dp), intent(in) :: noc, evals(:), dvals(:), beta + real(dp) :: mu + real(dp), allocatable :: fvals(:) + + if(.not.allocated(fvals))then + allocate(fvals(size(evals))) + endif + + if (present(verbose)) then + call gpmdcov_msI("gpmdcov_musearch_bisec","In gpmdcov_musearch_bisec ...",verbose,rank) + endif + + norb = size(evals,dim=1) + muMin=minval(evals) + muMax = maxval(evals) + mu = muMin + step=abs(muMax-muMin) + Ft1=0.0_dp + Ft2=0.0_dp + prod=0.0_dp + + !Sum of the occupations + call gpmdcov_fermifunction(beta,evals,mu,fvals) + + do i=1,norb + fermi = fvals(i) + ft1 = ft1 + 1.0_dp*fermi*dvals(i) + enddo + ft1=ft1-noc + + do m=1,1000001 + if(m.gt.1000000)then + write(*,*)"Bisection method in gpmdcov_musearch_bisec not converging ..." + stop + endif + if(mu > muMax + 1.0_dp .or. mu < muMin - 1.0_dp)then + write(*,*)"Bisection method is diverging" + write(*,*)"muMin=",muMin,"muMax=",muMax + write(*,*)evals + stop + endif + + if(abs(ft1).lt.tol)then !tolerance control + exit + endif + + mu = mu + step + + ft2=0.0_dp + + !New sum of the occupations + call gpmdcov_fermifunction(beta,evals,mu,fvals) + do i=1,norb + fermi = fvals(i) + ft2 = ft2 + 1.0_dp*fermi*dvals(i) + enddo + + ft2=ft2-noc + + !Product to see the change in sign. + prod = ft2*ft1 + if(prod.lt.0)then + mu=mu-step + step=step/2.0_dp !If the root is inside we shorten the step. + else + ft1=ft2 !If not, Ef moves forward. + endif + + enddo + + HOMO = muMin + LUMO = muMax + do i = 1,norb + if(evals(i) .lt. mu)then + HOMO = max(evals(i),HOMO) + elseif(evals(i) .gt. mu)then + LUMO = min(evals(i),LUMO) + endif + enddo + + !write(*,*)"EGAP BISEC",LUMO-HOMO,mu + egap = LUMO-HOMO + + end subroutine gpmdcov_musearch_bisec + +end module gpmdcov_mod diff --git a/examples/gpmdk/src/gpmdcov_mod_orig.F90 b/examples/gpmdk/src/gpmdcov_mod_orig.F90 new file mode 100644 index 00000000..43d993dc --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_mod_orig.F90 @@ -0,0 +1,91 @@ +!> Auxiliary modules. +!! \brief This module will be used to have auxiliary routines. +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_mod_orig + + implicit none + + private + + integer, parameter :: dp = kind(1.0d0) + + public :: gpmdcov_musearch + +contains + + + !> Perform musearch. + !! \param evals Eigenvalues of the system. + !! \para fvals Occupations of the states. + !! \param dvals Partial traces. + !! \prarm beta Electronic temperature. + !! \param maxMuIter Maximum numner of iterations. + !! \param occTol Occupation tolerance. + !! \param mu Chemical potential. + !! + subroutine gpmdcov_musearch(evals, fvals, dvals, beta, nocc, maxMuIter, occTol, mu, verbose) + implicit none + integer :: i, j, norbs + integer, intent(in) :: maxMuIter + real(dp) :: den, occErr, mu0, occ + real(dp), intent(in) :: evals(:), dvals(:) + real(dp), intent(inout) :: fvals(:) + real(dp), intent(in) :: occTol, beta, nocc + real(dp), intent(inout) :: mu + integer, optional, intent(in) :: verbose + + + norbs = size(evals, dim = 1) + + mu0 = mu + write(*,*)"NOCC",nocc, norbs, mu0 + + do j = 1, norbs + fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) + end do + + occ = 0.0_dp + do j = 1, norbs + ! write(*,*)"fvals, dvals",j,fvals(j),dvals(j), evals(j) + occ = occ + fvals(j)*dvals(j) + end do + + !stop + occErr = abs(occ-nocc) + + if (present(verbose)) then + if (verbose >= 1) write(*,*)"In gpmdcov_musearch ..." + endif + + do i = 1, maxMuIter + do j = 1, norbs + fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) + end do + + occ = 0.0_dp + den = 0.0_dp + do j = 1, norbs + occ = occ + fvals(j)*dvals(j) + den = den + beta*fvals(j)*(1.0_dp-fvals(j))*dvals(j) + end do + + occErr = abs(occ - nocc) + mu0 = mu0 + (nocc - occ)/den + mu = mu0 + + write(*,*)"occ, occErr, nocc, den, mu", occ, occErr, nocc, den, mu + if (i .eq. maxMuIter) then + write(*,*) "Max mu iters reached" + stop + else + if(occErr .lt. occTol) exit + end if + + end do + + end subroutine gpmdcov_musearch + +end module gpmdcov_mod_orig diff --git a/examples/gpmdk/src/gpmdcov_neighbor.F90 b/examples/gpmdk/src/gpmdcov_neighbor.F90 new file mode 100644 index 00000000..78b4a71d --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_neighbor.F90 @@ -0,0 +1,1445 @@ +module gpmdcov_neighbor_mod + + use bml + use prg_progress_mod + use prg_parallel_mod + use gpmdcov_writeout_mod + + implicit none + + private + + integer, parameter :: dp = kind(1.0d0) + integer, parameter :: low = kind(100) + + !> NL type + type, public :: neighlist_type !< The neighbor list type + + !> Number of atoms of the system. + integer :: nats + + !> Number of atoms within distance of Rcut from atom I including atoms in the skin + integer, allocatable :: nrnnlist(:) + + !> Distance between atom I(in box) and J (including atoms in the skin) + real(dp), allocatable :: nndist(:,:) + + !> x-coordinte of neighbor J to I within RCut (including atoms in the skin) + real(dp), allocatable :: nnRx(:,:) + !> y-coordinte of neighbor J to I within RCut (including atoms in the skin) + real(dp), allocatable :: nnRy(:,:) + !> z-coordinte of neighbor J to I within RCut (including atoms in the skin) + real(dp), allocatable :: nnRz(:,:) + + !> x-integer translation of neighbor J to I within RCut (including atoms in the skin) + integer, allocatable :: nnIx(:,:) + !> y-integer translation of neighbor J to I within RCut (including atoms in the skin) + integer, allocatable :: nnIy(:,:) + !> z-integer translation of neighbor J to I within RCut (including atoms in the skin) + integer, allocatable :: nnIz(:,:) + + !> The neighbor J of I corresponds to some translated atom number in the box that we need to keep track of. + integer, allocatable :: nnType(:,:) + + !> The neigbors J to I within Rcut that are all within the box (not in the skin). + integer, allocatable :: nnStruct(:,:) + + !> Number of neigbors to I within Rcut that are all within the box (not in the skin). + integer, allocatable :: nrnnStruct(:) + + !> Minimum distance between neighbors. nnStructMindist(i,j) means the Minimum distance + !! between nnStruct(i,j) and i considering translations. + real(dp), allocatable :: nnStructMindist(:,:) + + end type neighlist_type + + public :: gpmdcov_build_nlist_full, gpmdcov_destroy_nlist, gpmdcov_build_nlist_sparse + public :: gpmdcov_build_nlist_sparse_v2, gpmdcov_build_nlist_sparse_v3, gpmd_nearestneighborlist,gpmdcov_get_vol + +contains + + + !> Destroy the neigbor list to recover memory. + !! \param nl Neigbor list structure. + !! + subroutine gpmdcov_destroy_nlist(nl,verbose) + implicit none + type(neighlist_type), intent(inout) :: nl + integer, optional, intent(in) :: verbose + + if(present(verbose))then + if(verbose >= 1) write(*,*)"At gpmdcov_destroy_nlist. Destroying neighbor list ..." + endif + + if(allocated(nl%nrnnlist))deallocate(nl%nrnnlist) + if(allocated(nl%nndist))deallocate(nl%nndist) + if(allocated(nl%nnRx))deallocate(nl%nnRx) + if(allocated(nl%nnRy))deallocate(nl%nnRy) + if(allocated(nl%nnRz))deallocate(nl%nnRz) + if(allocated(nl%nnIx))deallocate(nl%nnIx) + if(allocated(nl%nnIy))deallocate(nl%nnIy) + if(allocated(nl%nnIz))deallocate(nl%nnIz) + if(allocated(nl%nnType))deallocate(nl%nnType) + if(allocated(nl%nnStruct))deallocate(nl%nnStruct) + if(allocated(nl%nrnnStruct))deallocate(nl%nrnnStruct) + if(allocated(nl%nnStructMindist))deallocate(nl%nnStructMindist) + + end subroutine gpmdcov_destroy_nlist + + !> Build the neighbor list with a "brute force method" + !! \brief It will bild a neighbor list using an "all to all" approach + !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. + !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. + !! \param nl Neighbor list type. + !! \param verbose Verbosity level. + !! \param rank MPI rank + subroutine gpmdcov_build_nlist_full(coords,lattice_vectors,rcut,nl,verbose,rank,numranks) + implicit none + integer :: cnt, i, j, k, maxneigh + integer :: myNumranks, myrank, nats, natsPerRank + integer, allocatable :: vectNnIx(:), vectNnIy(:), vectNnIz(:), vectNnStruct(:) + integer, allocatable :: vectNnType(:), vectNrnnStruct(:), vectNrnnlist(:) + integer, intent(in) :: verbose + integer, optional, intent(in) :: numranks, rank + real(dp) :: coordsNeigh(3), density, distance, translation(3) + real(dp) :: volBox, lvm(3), lvd(3,3) + real(dp), allocatable :: fcoords(:,:), fdvarray(:,:), dvarray(:,:), darray(:) + real(dp), allocatable, intent(in) :: coords(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: rcut + type(neighlist_type), intent(inout) :: nl +#ifdef DO_MPI + integer, allocatable :: rankRange(:,:) +#endif + + if(present(rank).and.present(numranks))then + myrank = rank + myNumranks = numranks + else + myrank = 1 + myNumranks = 1 + endif + + call gpmdcov_msI("gpmdcov_build_neigborlist","Building neighbor list ...",verbose,myrank) + + nats = size(coords,dim=2) !Get the number of atoms + + natsPerRank = int(nats/myNumranks) + +#ifdef DO_MPI + allocate(rankRange(2,myNumranks)) + do i = 1,myNumranks + rankRange(1,i) = (i-1)*natsPerRank + 1 + rankRange(2,i) = i*natsPerRank + enddo + rankRange(2,myNumranks) = rankRange(2,myNumranks) + (nats - myNumranks*natsPerRank) +#endif + + !We will have approximatly [pi * rcut^3 * atomic density] number of neighbors. + !This will always be bounded by [(2*rcut)^3 * atomic density] which we will use as + !our max number of neighbors. + call gpmdcov_get_vol(lattice_vectors,volBox) + density = nats/volBox + maxneigh = int(floor(8.0_dp * density * rcut**3)) + allocate(vectNnType(maxneigh*nats)) + vectNnType = 0 + allocate(vectNnStruct(maxneigh*nats)) + vectNnStruct = 0 + allocate(vectNrnnStruct(nats)) + vectNrnnStruct = 0 + allocate(vectNrnnlist(nats)) + vectNrnnlist = 0 + + allocate(fcoords(nats,3)) + allocate(fdvarray(nats,3)) + allocate(dvarray(nats,3)) + allocate(darray(nats)) + + do i = 1,3 + lvm(i) = sqrt(lattice_vectors(i,1)*lattice_vectors(i,1) & + + lattice_vectors(i,2)*lattice_vectors(i,2) & + + lattice_vectors(i,3)*lattice_vectors(i,3)) + lvd(i,:) = lattice_vectors(i,:)/lvm(i) + fcoords(:,i) = (lvd(i,1)*coords(1,:)+lvd(i,2)*coords(2,:)+lvd(i,3)*coords(3,:))/lvm(i) + end do + + !$omp parallel do default(none) private(i) & + !$omp private(cnt,j) & + !$omp private(distance,translation,coordsNeigh) & + !$omp private(dvarray, darray, fdvarray) & + !$omp shared(fcoords,coords,rcut,vectNnType) & + !$omp shared(verbose,vectNnIx,vectNnIy,vectNnIz) & + !$omp shared(vectNnStruct,vectNrnnStruct,vectNrnnlist) & + !$omp shared(lattice_vectors)& + !$omp shared(maxneigh) & +#ifdef DO_MPI + !$omp shared(nats,rankRange,myRank) + do i = rankRange(1,myRank),rankRange(2,myRank) !For every atom in the rank range +#else + !$omp shared(nats) + do i = 1,nats !For every atom +#endif + do k = 1,3 + fdvarray(:,k) = modulo((fcoords(:,k) - fcoords(i,k)) + 0.5,1.) - 0.5 + enddo + + dvarray = matmul(fdvarray,lattice_vectors) + darray = norm2(dvarray,DIM=2) + + cnt = 0 + do j = 1,nats !For every of its possible neighbors + if (darray(j) .lt. rcut .and. darray(j) .gt. 1d-12) then + cnt = cnt + 1 + vectNntype((i-1)*maxneigh + cnt) = j ! j is a neighbor of i by some translation + vectNnstruct((i-1)*maxneigh + cnt) = j ! j is a neighbor of i by some translation + endif + enddo + vectNrnnStruct(i) = cnt + vectNrnnlist(i) = cnt + enddo + !$omp end parallel do + + !We do a sum reduction on all the vectors +#ifdef DO_MPI + call prg_sumIntReduceN(vectNntype,maxneigh*nats) + call prg_sumIntReduceN(vectNnstruct,maxneigh*nats) + call prg_sumIntReduceN(vectNrnnStruct,nats) + call prg_sumIntReduceN(vectNrnnlist,nats) +#endif + + !We deallocate the auxiliary vectors and transform them to matrices. + if(.not.allocated(nl%nnType))allocate(nl%nnType(maxneigh,nats)) + call vectorToMatrixInt(maxneigh,nats,vectNnType,nl%nnType) + deallocate(vectNnType) + + if(.not.allocated(nl%nnStruct))allocate(nl%nnStruct(maxneigh,nats)) + call vectorToMatrixInt(maxneigh,nats,vectNnStruct,nl%nnStruct) + deallocate(vectNnStruct) + + if(.not.allocated(nl%nrnnStruct))allocate(nl%nrnnStruct(nats)) + nl%nrnnStruct = vectNrnnStruct + deallocate(vectNrnnStruct) + + if(.not.allocated(nl%nrnnlist))allocate(nl%nrnnlist(nats)) + nl%nrnnlist = vectNrnnlist + deallocate(vectNrnnlist) + +#ifdef DO_MPI + deallocate(rankRange) +#endif + + deallocate(fcoords) + deallocate(fdvarray) + deallocate(darray) + deallocate(dvarray) + + end subroutine gpmdcov_build_nlist_full + + + !> Build the neighbor list with a "brute force method" + !! \brief It will bild a neighbor list using an "all to all" approach + !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. + !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. + !! \param nl Neighbor list type. + !! \param verbose Verbosity level. + !! \param rank MPI rank + subroutine gpmdcov_build_nlist_sparse(coords,lattice_vectors,rcut,nl,verbose,rank,numranks) + implicit none + integer :: NBox, cnt, i, ibox + integer :: ith, ix, iy, iz + integer :: j, jbox, jj, jxBox + integer :: jyBox, jzBox, maxInBox, maxNeigh + integer :: myNumranks, myrank, nats, natsPerRank + integer :: nx, ny, nz, tx + integer :: ty, tz + integer, allocatable :: boxOfI(:), inbox(:,:), ithFromXYZ(:,:,:) + integer, allocatable :: totPerBox(:), xBox(:), yBox(:), zBox(:) + integer, allocatable :: vectNnIx(:), vectNnIy(:), vectNnIz(:), vectNnStruct(:) + integer, allocatable :: vectNnType(:), vectNrnnStruct(:), vectNrnnlist(:) + integer, intent(in) :: verbose + integer, optional, intent(in) :: numranks, rank + real(dp) :: coordsNeigh(3), density, distance, translation(3) + real(dp) :: volBox, minx, miny, minz, smallReal + real(dp), allocatable, intent(in) :: coords(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: rcut + type(neighlist_type), intent(inout) :: nl +#ifdef DO_MPI + integer, allocatable :: rankRange(:,:) +#endif + + if(present(rank).and.present(numranks))then + myrank = rank + myNumranks = numranks + else + myrank = 1 + myNumranks = 1 + endif + + call gpmdcov_msI("gpmdcov_build_neigborlist","Building neighbor list ...",verbose,myrank) + + nats = size(coords,dim=2) !Get the number of atoms + natsPerRank = int(nats/myNumranks) + +#ifdef DO_MPI + allocate(rankRange(2,myNumranks)) + do i = 1,myNumranks + rankRange(1,i) = (i-1)*natsPerRank + 1 + rankRange(2,i) = i*natsPerRank + enddo + rankRange(2,myNumranks) = rankRange(2,myNumranks) + (nats - myNumranks*natsPerRank) +#endif + + !We will have approximatly [(4/3)*pi * rcut^3 * atomic density] number of neighbors. + !A very large atomic density could be 1 atom per (1.0 Ang)^3 = 1 atoms per Ang^3 + call gpmdcov_get_vol(lattice_vectors,volBox) + density = 1.0_dp + maxneigh = int(floor(3.14592_dp * 4.0_dp/3.0_dp * density * rcut**3)) + + !We assume the box is orthogonal + nx = 1 + floor(lattice_vectors(1,1)/rcut) + ny = 1 + floor(lattice_vectors(2,2)/rcut) + nz = 1 + floor(lattice_vectors(3,3)/rcut) + + NBox = nx*ny*nz + maxInBox = int(density*rcut**3) !Upper boud for the max number of atoms per box + + allocate(inbox(NBox,maxInBox)) + inbox = 0 + allocate(totPerBox(Nbox)) + totPerBox = 0 + allocate(boxOfI(nats)) + boxOfI = 0 + allocate(xBox(Nbox)) + xBox = 0 + allocate(yBox(Nbox)) + yBox = 0 + allocate(zBox(Nbox)) + zBox = 0 + allocate(ithFromXYZ(nx,ny,nz)) + ithFromXYZ = 0 + + allocate(vectNnType(maxneigh*nats)) + vectNnType = 0 + allocate(vectNnIx(maxneigh*nats)) + vectNnIx = 0 + allocate(vectNnIy(maxneigh*nats)) + vectNnIy = 0 + allocate(vectNnIz(maxneigh*nats)) + vectNnIz = 0 + allocate(vectNnStruct(maxneigh*nats)) + vectNnStruct = 0 + allocate(vectNrnnStruct(nats)) + vectNrnnStruct = 0 + allocate(vectNrnnlist(nats)) + vectNrnnlist = 0 + + minx = 1.0d10 + miny = 1.0d10 + minz = 1.0d10 + do i = 1,nats + minx = min(minx,coords(1,i)) + miny = min(miny,coords(2,i)) + minz = min(minz,coords(3,i)) + enddo + + smallReal = 0.0_dp + !Search for the box coordinate and index of every atom + do i = 1,nats + !Index every atom respect to the discretized position on the simulation box. + !tranlation = coords(:,i) - origin !For the general case we need to make sure coords ar > 0 + ix = 1+ int(floor((coords(1,i) - minx + smallReal)/rcut)) !small box x-index of atom i + iy = 1+ int(floor((coords(2,i) - miny + smallReal)/rcut)) !small box y-index // + iz = 1+ int(floor((coords(3,i) - minz + smallReal)/rcut)) !small box z-index // + + if(ix > nx .or. ix < 0)Stop "Error in box index" + if(iy > ny .or. iy < 0)Stop "Error in box index" + if(iz > nz .or. iz < 0)Stop "Error in box index" + + ith = ix + (iy-1)*(nx) + (iz-1)*(nx)*(ny) !Get small box index + boxOfI(i) = ith + + !From index to box coordinates + xBox(ith) = ix + yBox(ith) = iy + zBox(ith) = iz + + !From box coordinates to index + ithFromXYZ(ix,iy,iz) = ith + + totPerBox(ith) = totPerBox(ith) + 1 !How many per box + if(totPerBox(ith) > maxInBox) Stop "Exceeding the max in box allowed" + inbox(ith,totPerBox(ith)) = i !Who is in both ith + + enddo + + !For each atom we will look around to see who are its neighbors + !$omp parallel do default(none) private(i) & + !$omp private(ibox,ix,iy,iz) & + !$omp private(jxbox,jybox,jzbox,jbox) & + !$omp private(distance,translation,coordsNeigh) & + !$omp private(cnt,j,jj,tx,ty,tz) & + !$omp shared(xbox,ybox,zbox,boxOfI) & + !$omp shared(nx,ny,nz,inbox,totperbox) & + !$omp shared(coords,rcut,vectNnType,ithfromxyz) & + !$omp shared(vectNnIx,vectNnIy,vectNnIz) & + !$omp shared(vectNnStruct,vectNrnnStruct,vectNrnnlist) & + !$omp shared(lattice_vectors)& + !$omp shared(maxneigh) & +#ifdef DO_MPI + !$omp shared(rankRange,myRank,nats) + do i = rankRange(1,myRank),rankRange(2,myRank) !For every atom in the rank range +#else + !$omp shared(nats) + do i = 1,nats !For every atom +#endif + cnt = 0 + !Which box it beongs to + ibox = boxOfI(i) + !Look inside the box and the neighboring boxes + do ix = -1,1 + do iy = -1,1 + do iz = -1,1 + !Get neigh box coordinate + jxBox = xBox(ibox) + ix + jyBox = yBox(ibox) + iy + jzBox = zBox(ibox) + iz + tx = 0.0_dp ; ty = 0.0_dp ; tz = 0.0_dp + if(jxBox <= 0)then + jxBox = nx + tx = -1 + elseif(jxBox > nx)then + jxBox = 1 + tx = 1 + endif + if(jyBox <= 0)then + jyBox = ny + ty = -1 + elseif(jyBox > ny)then + jyBox = 1 + ty = 1 + endif + if(jzBox <= 0)then + jzBox = nz + tz = -1 + elseif(jzBox > nz)then + jzBox = 1 + tz = 1 + endif + + !Get the neigh box index + jbox = ithFromXYZ(jxBox,jyBox,jzBox) + + !Now loop over the atoms in the jbox + do j = 1,totPerBox(jbox) + jj = inbox(jbox,j) !Get atoms in box j + translation = tx*lattice_vectors(1,:) + ty*lattice_vectors(2,:) + tz*lattice_vectors(3,:) + coordsNeigh = coords(:,jj) + translation + distance = norm2(coords(:,i) - coordsNeigh) + if (distance .lt. rcut .and. distance .gt. 1d-12) then + cnt = cnt + 1 + vectNntype((i-1)*maxneigh + cnt) = jj ! jj is a neighbor of i by some translation + vectNnstruct((i-1)*maxneigh + cnt) = jj ! jj is a neighbor of i by some translation + vectNnIx((i-1)*maxneigh + cnt) = tx + vectNnIy((i-1)*maxneigh + cnt) = ty + vectNnIz((i-1)*maxneigh + cnt) = tz + + endif + enddo + enddo + enddo + enddo + vectNrnnStruct(i) = cnt + vectNrnnlist(i) = cnt + enddo +!$omp end parallel do + + deallocate(inbox) + deallocate(totPerBox) + deallocate(boxOfI) + deallocate(xBox) + deallocate(yBox) + deallocate(zBox) + deallocate(ithFromXYZ) + + + !We do a sum reduction on all the vectors +#ifdef DO_MPI + call prg_sumIntReduceN(vectNntype,maxneigh*nats) + call prg_sumIntReduceN(vectNnstruct,maxneigh*nats) + call prg_sumIntReduceN(vectNnIx,maxneigh*nats) + call prg_sumIntReduceN(vectNnIy,maxneigh*nats) + call prg_sumIntReduceN(vectNnIz,maxneigh*nats) + call prg_sumIntReduceN(vectNrnnStruct,nats) + call prg_sumIntReduceN(vectNrnnlist,nats) +#endif + + !We deallocate the auxiliary vectors and transform them to matrices. + if(.not.allocated(nl%nnType))allocate(nl%nnType(maxneigh,nats)) + call vectorToMatrixInt(maxneigh,nats,vectNnType,nl%nnType) + deallocate(vectNnType) + + if(.not.allocated(nl%nnIx))allocate(nl%nnIx(maxneigh,nats)) + call vectorToMatrixIntLow(maxneigh,nats,vectNnIx,nl%nnIx) + deallocate(vectNnIx) + + if(.not.allocated(nl%nnIy))allocate(nl%nnIy(maxneigh,nats)) + call vectorToMatrixIntLow(maxneigh,nats,vectNnIy,nl%nnIy) + deallocate(vectNnIy) + + if(.not.allocated(nl%nnIz))allocate(nl%nnIz(maxneigh,nats)) + call vectorToMatrixIntLow(maxneigh,nats,vectNnIz,nl%nnIz) + deallocate(vectNnIz) + + if(.not.allocated(nl%nnStruct))allocate(nl%nnStruct(maxneigh,nats)) + call vectorToMatrixInt(maxneigh,nats,vectNnStruct,nl%nnStruct) + deallocate(vectNnStruct) + + if(.not.allocated(nl%nrnnStruct))allocate(nl%nrnnStruct(nats)) + nl%nrnnStruct = vectNrnnStruct + deallocate(vectNrnnStruct) + + if(.not.allocated(nl%nrnnlist))allocate(nl%nrnnlist(nats)) + nl%nrnnlist = vectNrnnlist + deallocate(vectNrnnlist) + +#ifdef DO_MPI + deallocate(rankRange) +#endif + + end subroutine gpmdcov_build_nlist_sparse + + !> Convert a vector to a matrix. + !! \param rows Number of rows. + !! \param cols Number of columns. + !! \param vect Vector input. + !! \param mat Matix output. + subroutine vectorToMatrixIntLow(rows,cols,vect,mat) + implicit none + integer, allocatable, intent(in) :: vect(:) + integer(low), allocatable, intent(inout) :: mat(:,:) + integer, intent(in) :: rows, cols + integer :: i,j + do j = 1,cols + do i = 1,rows + mat(i,j) = vect((j-1)*rows + i) + enddo + enddo + end subroutine vectorToMatrixIntLow + + !> Convert a vector to a matrix. + !! \param rows Number of rows. + !! \param cols Number of columns. + !! \param vect Vector input. + !! \param mat Matix output. + subroutine vectorToMatrixInt(rows,cols,vect,mat) + implicit none + integer, allocatable, intent(in) :: vect(:) + integer, allocatable, intent(inout) :: mat(:,:) + integer, intent(in) :: rows, cols + integer :: i,j + do j = 1,cols + do i = 1,rows + mat(i,j) = vect((j-1)*rows + i) + enddo + enddo + end subroutine vectorToMatrixInt + + !> Gets the volume of the simulation box + !! \brief Given an array of lattice vectors, it return the box volume + !! \param lattice_vector Lattice vectors in an array. latice_vectors(1,3) means the z-coordinate + !! of the first lattice vector. + !! \param volBox Volume of the cell. + subroutine gpmdcov_get_vol(lattice_vectors,volBox) + implicit none + real(dp) :: a1xa2(3), a2xa3(3), a3xa1(3) + real(dp) :: pi + real(dp), intent(in) :: lattice_vectors(:,:) + real(dp), intent(inout) :: volBox + + volBox=0.0_dp + + pi = 3.14159265358979323846264338327950_dp + + a1xa2(1) = lattice_vectors(1,2)*lattice_vectors(2,3) - lattice_vectors(1,3)*lattice_vectors(2,2) + a1xa2(2) = -lattice_vectors(1,1)*lattice_vectors(2,3) + lattice_vectors(1,3)*lattice_vectors(2,1) + a1xa2(3) = lattice_vectors(1,1)*lattice_vectors(2,2) - lattice_vectors(1,2)*lattice_vectors(2,1) + + a2xa3(1) = lattice_vectors(2,2)*lattice_vectors(3,3) - lattice_vectors(2,3)*lattice_vectors(3,2) + a2xa3(2) = -lattice_vectors(2,1)*lattice_vectors(3,3) + lattice_vectors(2,3)*lattice_vectors(3,1) + a2xa3(3) = lattice_vectors(2,1)*lattice_vectors(3,2) - lattice_vectors(2,2)*lattice_vectors(3,1) + + a3xa1(1) = lattice_vectors(3,2)*lattice_vectors(1,3) - lattice_vectors(3,3)*lattice_vectors(1,2) + a3xa1(2) = -lattice_vectors(3,1)*lattice_vectors(1,3) + lattice_vectors(3,3)*lattice_vectors(1,1) + a3xa1(3) = lattice_vectors(3,1)*lattice_vectors(1,2) - lattice_vectors(3,2)*lattice_vectors(1,1) + + !Get the volume of the cell + volBox = lattice_vectors(1,1)*a2xa3(1)+ lattice_vectors(1,2)*a2xa3(2)+lattice_vectors(1,3)*a2xa3(3) + + end subroutine gpmdcov_get_vol + + + !! \brief It will bild a neighbor list using an "all to all" approach + !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. + !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. + !! \param nl Neighbor list type. + !! \param verbose Verbosity level. + !! \param rank MPI rank + subroutine gpmdcov_build_nlist_sparse_v2(coords,lattice_vectors,rcut,nl,verbose,rank,numranks) + implicit none + integer :: NBox, cnt, i, ibox + integer :: ith, ix, iy, iz + integer :: j, jbox, jj, jxBox + integer :: jyBox, jzBox, maxInBox, maxNeigh + integer :: myNumranks, myrank, nats, natsPerRank + integer :: nx, ny, nz, tx + integer :: ty, tz + integer, allocatable :: boxOfI(:), inbox(:,:), ithFromXYZ(:,:,:) + integer, allocatable :: totPerBox(:), xBox(:), yBox(:), zBox(:) + integer, intent(in) :: verbose + integer, optional, intent(in) :: numranks, rank + real(dp) :: coordsNeigh(3), density, distance, translation(3) + real(dp) :: volBox, minx, miny, minz, smallReal, mlsnl + real(dp), allocatable, intent(in) :: coords(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: rcut + type(neighlist_type), intent(inout) :: nl +#ifdef DO_MPI + integer, allocatable :: rankRange(:,:) +#endif + + if(present(rank).and.present(numranks))then + myrank = rank + myNumranks = numranks + else + myrank = 1 + myNumranks = 1 + endif + + call gpmdcov_msI("gpmdcov_build_neigborlist","Building neighbor list ...",verbose,myrank) + + nats = size(coords,dim=2) !Get the number of atoms + natsPerRank = int(nats/myNumranks) + + !We will have approximatly [(4/3)*pi * rcut^3 * atomic density] number of neighbors. + !A very large atomic density could be 1 atom per (1.0 Ang)^3 = 1 atoms per Ang^3 + call gpmdcov_get_vol(lattice_vectors,volBox) + density = 1.0_dp + maxneigh = int(floor(3.14592_dp * (4.0_dp/3.0_dp) * density * rcut**3)) + + !We assume the box is orthogonal + nx = 1 + floor(lattice_vectors(1,1)/(2.0_dp*rcut)) + ny = 1 + floor(lattice_vectors(2,2)/(2.0_dp*rcut)) + nz = 1 + floor(lattice_vectors(3,3)/(2.0_dp*rcut)) + !write(*,*)"nx,ny,nz",nx,ny,nz + !stop + NBox = nx*ny*nz + maxInBox = int(density*rcut**3) !Upper boud for the max number of atoms per box + mlsnl = mls() + allocate(inbox(NBox,maxInBox)) + inbox = 0 + allocate(totPerBox(Nbox)) + totPerBox = 0 + allocate(boxOfI(nats)) + boxOfI = 0 + allocate(xBox(Nbox)) + xBox = 0 + allocate(yBox(Nbox)) + yBox = 0 + allocate(zBox(Nbox)) + zBox = 0 + allocate(ithFromXYZ(nx,ny,nz)) + ithFromXYZ = 0 + + minx = 1.0d10 + miny = 1.0d10 + minz = 1.0d10 + do i = 1,nats + minx = min(minx,coords(1,i)) + miny = min(miny,coords(2,i)) + minz = min(minz,coords(3,i)) + enddo + + smallReal = 0.0_dp + !Search for the box coordinate and index of every atom + do i = 1,nats + !Index every atom respect to the discretized position on the simulation box. + !tranlation = coords(:,i) - origin !For the general case we need to make sure coords ar > 0 + ix = 1+ int(floor((coords(1,i) - minx + smallReal)/(2.0_dp*rcut))) !small box x-index of atom i + iy = 1+ int(floor((coords(2,i) - miny + smallReal)/(2.0_dp*rcut))) !small box y-index // + iz = 1+ int(floor((coords(3,i) - minz + smallReal)/(2.0_dp*rcut))) !small box z-index // + + if(ix > nx .or. ix < 0)Stop "Error in box index" + if(iy > ny .or. iy < 0)Stop "Error in box index" + if(iz > nz .or. iz < 0)Stop "Error in box index" + + ith = ix + (iy-1)*(nx) + (iz-1)*(nx)*(ny) !Get small box index + boxOfI(i) = ith + + !From index to box coordinates + xBox(ith) = ix + yBox(ith) = iy + zBox(ith) = iz + + !From box coordinates to index + ithFromXYZ(ix,iy,iz) = ith + + totPerBox(ith) = totPerBox(ith) + 1 !How many per box + if(totPerBox(ith) > maxInBox) Stop "Exceeding the max in box allowed" + inbox(ith,totPerBox(ith)) = i !Who is in both ith + + enddo + + if(.not.allocated(nl%nnType))allocate(nl%nnType(maxneigh,nats)) + if(.not.allocated(nl%nnIx))allocate(nl%nnIx(maxneigh,nats)) + if(.not.allocated(nl%nnIy))allocate(nl%nnIy(maxneigh,nats)) + if(.not.allocated(nl%nnIz))allocate(nl%nnIz(maxneigh,nats)) + if(.not.allocated(nl%nnStruct))allocate(nl%nnStruct(maxneigh,nats)) + if(.not.allocated(nl%nrnnStruct))allocate(nl%nrnnStruct(nats)) + if(.not.allocated(nl%nrnnlist))allocate(nl%nrnnlist(nats)) + + + !For each atom we will look around to see who are its neighbors + !$omp parallel do default(none) private(i) & + !$omp private(ibox,ix,iy,iz) & + !$omp private(jxbox,jybox,jzbox,jbox) & + !$omp private(distance,translation,coordsNeigh) & + !$omp private(cnt,j,jj,tx,ty,tz) & + !$omp shared(nx,ny,nz,boxOfI) & + !$omp shared(xBox,yBox,zBox) & + !$omp shared(coords,rcut,totPerBox) & + !$omp shared(nl,inbox,ithFromXYZ) & + !$omp shared(lattice_vectors)& + !$omp shared(maxneigh) & + !$omp shared(nats) + do i = 1,nats !For every atom +!#endif + cnt = 0 + !Which box it beongs to + ibox = boxOfI(i) + !Look inside the box and the neighboring boxes + do ix = -1,1 + do iy = -1,1 + do iz = -1,1 + !Get neigh box coordinate + jxBox = xBox(ibox) + ix + jyBox = yBox(ibox) + iy + jzBox = zBox(ibox) + iz + tx = 0.0_dp ; ty = 0.0_dp ; tz = 0.0_dp + if(jxBox <= 0)then + jxBox = nx + tx = -1 + elseif(jxBox > nx)then + jxBox = 1 + tx = 1 + endif + if(jyBox <= 0)then + jyBox = ny + ty = -1 + elseif(jyBox > ny)then + jyBox = 1 + ty = 1 + endif + if(jzBox <= 0)then + jzBox = nz + tz = -1 + elseif(jzBox > nz)then + jzBox = 1 + tz = 1 + endif + + !Get the neigh box index + jbox = ithFromXYZ(jxBox,jyBox,jzBox) + + !Now loop over the atoms in the jbox + do j = 1,totPerBox(jbox) + jj = inbox(jbox,j) !Get atoms in box j + translation = tx*lattice_vectors(1,:) + ty*lattice_vectors(2,:) + tz*lattice_vectors(3,:) + coordsNeigh = coords(:,jj) + translation + distance = norm2(coords(:,i) - coordsNeigh) + if (distance .lt. rcut .and. distance .gt. 1d-12) then + cnt = cnt + 1 + nl%Nntype(cnt,i) = jj ! jj is a neighbor of i by some translation + nl%Nnstruct(cnt,i) = jj ! jj is a neighbor of i by some translation + nl%NnIx(cnt,i) = tx + nl%NnIy(cnt,i) = ty + nl%NnIz(cnt,i) = tz + endif + enddo + enddo + enddo + enddo + nl%NrnnStruct(i) = cnt + nl%Nrnnlist(i) = cnt + + enddo + !$omp end parallel do + + deallocate(inbox) + deallocate(totPerBox) + deallocate(boxOfI) + deallocate(xBox) + deallocate(yBox) + deallocate(zBox) + deallocate(ithFromXYZ) + + end subroutine gpmdcov_build_nlist_sparse_v2 + + + !> Build the neighbor list with a "brute force method" + !! \brief It will bild a neighbor list using an "all to all" approach + !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. + !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. + !! \param nl Neighbor list type. + !! \param verbose Verbosity level. + !! \param rank MPI rank + subroutine gpmdcov_build_nlist_sparse_v3(coords,lattice_vectors,rcut,nl,verbose,rank,numranks) + implicit none + integer :: NBox, cnt, i, ibox + integer :: ith, ix, iy, iz + integer :: j, jbox, jj, jxBox + integer :: jyBox, jzBox, maxInBox, maxNeigh + integer :: myNumranks, myrank, nats, natsPerRank + integer :: nx, ny, nz, tx + integer :: ty, tz + integer, allocatable :: boxOfI(:), inbox(:,:), ithFromXYZ(:,:,:) + integer, allocatable :: totPerBox(:), xBox(:), yBox(:), zBox(:) + integer, intent(in) :: verbose + integer, optional, intent(in) :: numranks, rank + real(dp) :: coordsNeigh(3), density, distance, translation(3) + real(dp) :: volBox, minx, miny, minz, smallReal, mlsnl + real(dp), allocatable, intent(in) :: coords(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: rcut + type(neighlist_type), intent(inout) :: nl +#ifdef DO_MPI + integer, allocatable :: rankRange(:,:) +#endif + + if(present(rank).and.present(numranks))then + myrank = rank + myNumranks = numranks + else + myrank = 1 + myNumranks = 1 + endif + + call gpmdcov_msI("gpmdcov_build_neigborlist","Building neighbor list ...",verbose,myrank) + + nats = size(coords,dim=2) !Get the number of atoms + natsPerRank = int(nats/myNumranks) + +#ifdef DO_MPI + allocate(rankRange(2,myNumranks)) + do i = 1,myNumranks + rankRange(1,i) = (i-1)*natsPerRank + 1 + rankRange(2,i) = i*natsPerRank + enddo + rankRange(2,myNumranks) = rankRange(2,myNumranks) + (nats - myNumranks*natsPerRank) +#endif + + + !We will have approximatly [(4/3)*pi * rcut^3 * atomic density] number of neighbors. + !A very large atomic density could be 1 atom per (1.0 Ang)^3 = 1 atoms per Ang^3 + call gpmdcov_get_vol(lattice_vectors,volBox) + density = 1.0_dp + maxneigh = int(floor(3.14592_dp * 4.0_dp/3.0_dp * density * rcut**3)) + + !We assume the box is orthogonal + nx = 1 + floor(lattice_vectors(1,1)/rcut) + ny = 1 + floor(lattice_vectors(2,2)/rcut) + nz = 1 + floor(lattice_vectors(3,3)/rcut) + + NBox = nx*ny*nz + maxInBox = int(density*rcut**3) !Upper boud for the max number of atoms per box + mlsnl = mls() + allocate(inbox(NBox,maxInBox)) + inbox = 0 + allocate(totPerBox(Nbox)) + totPerBox = 0 + allocate(boxOfI(nats)) + boxOfI = 0 + allocate(xBox(Nbox)) + xBox = 0 + allocate(yBox(Nbox)) + yBox = 0 + allocate(zBox(Nbox)) + zBox = 0 + allocate(ithFromXYZ(nx,ny,nz)) + ithFromXYZ = 0 + + minx = 1.0d10 + miny = 1.0d10 + minz = 1.0d10 + do i = 1,nats + minx = min(minx,coords(1,i)) + miny = min(miny,coords(2,i)) + minz = min(minz,coords(3,i)) + enddo + + smallReal = 0.0_dp + write(*,*)"nlist Time allocs",mls() - mlsnl + mlsnl = mls() + !Search for the box coordinate and index of every atom + do i = 1,nats + !Index every atom respect to the discretized position on the simulation box. + !tranlation = coords(:,i) - origin !For the general case we need to make sure coords ar > 0 + ix = 1+ int(floor((coords(1,i) - minx + smallReal)/rcut)) !small box x-index of atom i + iy = 1+ int(floor((coords(2,i) - miny + smallReal)/rcut)) !small box y-index // + iz = 1+ int(floor((coords(3,i) - minz + smallReal)/rcut)) !small box z-index // + + if(ix > nx .or. ix < 0)Stop "Error in box index" + if(iy > ny .or. iy < 0)Stop "Error in box index" + if(iz > nz .or. iz < 0)Stop "Error in box index" + + ith = ix + (iy-1)*(nx) + (iz-1)*(nx)*(ny) !Get small box index + boxOfI(i) = ith + + !From index to box coordinates + xBox(ith) = ix + yBox(ith) = iy + zBox(ith) = iz + + !From box coordinates to index + ithFromXYZ(ix,iy,iz) = ith + + totPerBox(ith) = totPerBox(ith) + 1 !How many per box + if(totPerBox(ith) > maxInBox) Stop "Exceeding the max in box allowed" + inbox(ith,totPerBox(ith)) = i !Who is in both ith + + enddo + write(*,*)"nlist Time seting up boxes",mls() - mlsnl + mlsnl = mls() + + if(.not.allocated(nl%nnType))allocate(nl%nnType(maxneigh,nats)) + if(.not.allocated(nl%nnIx))allocate(nl%nnIx(maxneigh,nats)) + if(.not.allocated(nl%nnIy))allocate(nl%nnIy(maxneigh,nats)) + if(.not.allocated(nl%nnIz))allocate(nl%nnIz(maxneigh,nats)) + if(.not.allocated(nl%nnStruct))allocate(nl%nnStruct(maxneigh,nats)) + if(.not.allocated(nl%nrnnStruct))allocate(nl%nrnnStruct(nats)) + if(.not.allocated(nl%nrnnlist))allocate(nl%nrnnlist(nats)) + + + !For each atom we will look around to see who are its neighbors + !$omp parallel do default(none) private(i) & + !$omp private(ibox,ix,iy,iz) & + !$omp private(jxbox,jybox,jzbox,jbox) & + !$omp private(distance,translation,coordsNeigh) & + !$omp private(cnt,j,jj,tx,ty,tz) & + !$omp shared(nx,ny,nz,inbox) & + !$omp shared(coords,rcut,ithFromXYZ,boxOfI,totPerBox) & + !$omp shared(nl,xBox,yBox,zBox) & + !$omp shared(lattice_vectors)& + !$omp shared(maxneigh) & +#ifdef DO_MPI + !$omp shared(rankRange,myRank,nats) + do i = rankRange(1,myRank),rankRange(2,myRank) !For every atom in the rank range +#else + !$omp shared(nats) + do i = 1,nats !For every atom +#endif + cnt = 0 + !Which box it beongs to + ibox = boxOfI(i) + !Look inside the box and the neighboring boxes + do ix = -1,1 + do iy = -1,1 + do iz = -1,1 + !Get neigh box coordinate + jxBox = xBox(ibox) + ix + jyBox = yBox(ibox) + iy + jzBox = zBox(ibox) + iz + tx = 0.0_dp ; ty = 0.0_dp ; tz = 0.0_dp + if(jxBox <= 0)then + jxBox = nx + tx = -1 + elseif(jxBox > nx)then + jxBox = 1 + tx = 1 + endif + if(jyBox <= 0)then + jyBox = ny + ty = -1 + elseif(jyBox > ny)then + jyBox = 1 + ty = 1 + endif + if(jzBox <= 0)then + jzBox = nz + tz = -1 + elseif(jzBox > nz)then + jzBox = 1 + tz = 1 + endif + + !Get the neigh box index + jbox = ithFromXYZ(jxBox,jyBox,jzBox) + + !Now loop over the atoms in the jbox + do j = 1,totPerBox(jbox) + jj = inbox(jbox,j) !Get atoms in box j + translation = tx*lattice_vectors(1,:) + ty*lattice_vectors(2,:) + tz*lattice_vectors(3,:) + coordsNeigh = coords(:,jj) + translation + distance = norm2(coords(:,i) - coordsNeigh) + if (distance .lt. rcut .and. distance .gt. 1d-12) then + cnt = cnt + 1 + nl%Nntype(cnt,i) = jj ! jj is a neighbor of i by some translation + nl%Nnstruct(cnt,i) = jj ! jj is a neighbor of i by some translation + nl%NnIx(cnt,i) = tx + nl%NnIy(cnt,i) = ty + nl%NnIz(cnt,i) = tz + endif + enddo + enddo + enddo + enddo + nl%NrnnStruct(i) = cnt + nl%Nrnnlist(i) = cnt + + enddo + !$omp end parallel do + write(*,*)"nlist Time nlist rsearch",mls() - mlsnl + + deallocate(inbox) + deallocate(totPerBox) + deallocate(boxOfI) + deallocate(xBox) + deallocate(yBox) + deallocate(zBox) + deallocate(ithFromXYZ) + + !We do a sum reduction on all the vectors +#ifdef DO_MPI + call prg_sumIntReduceN(nl%Nntype,maxneigh*nats) + call prg_sumIntReduceN(nl%Nnstruct,maxneigh*nats) + call prg_sumIntReduceN(nl%NnIx,maxneigh*nats) + call prg_sumIntReduceN(nl%NnIy,maxneigh*nats) + call prg_sumIntReduceN(nl%NnIz,maxneigh*nats) + call prg_sumIntReduceN(nl%NrnnStruct,nats) + call prg_sumIntReduceN(nl%Nrnnlist,nats) +#endif + + write(*,*)"nlist Time for transfer",mls() - mlsnl + + end subroutine gpmdcov_build_nlist_sparse_v3 + + + subroutine gpmd_nearestneighborlist(nrnnlist,nndist,nnRx,nnRy,nnRz,nnType, & + &R_X,R_Y,R_Z,LBox,Rcut,Nr_atoms,Max_Nr_Neigh) +implicit none + +integer, parameter :: MSkin = 2 +real(dp), parameter :: ONE = 1.D0, TWO = 2.D0, ZERO = 0.D0 +integer, intent(in) :: Nr_atoms, Max_Nr_Neigh +real(dp), intent(in) :: Rcut +real(dp), intent(in) :: R_X(Nr_atoms), R_Y(Nr_atoms), R_Z(Nr_atoms), LBox(3) +real(dp) :: Rx(MSkin*Nr_atoms), Ry(MSkin*Nr_atoms), Rz(MSkin*Nr_atoms) +integer, intent(out) :: nrnnlist(Nr_atoms), nnType(Max_Nr_Neigh,Nr_atoms) +real(dp), intent(out) :: nndist(Max_Nr_Neigh,Nr_atoms), nnRx(Max_Nr_Neigh,Nr_atoms) +real(dp), intent(out) :: nnRy(Max_Nr_Neigh,Nr_atoms), nnRz(Max_Nr_Neigh,Nr_atoms) +integer :: i,j,k,l,m,t,nx,ny,nz,cell,type(10*Nr_atoms),Nskin,N +integer :: head(Nr_atoms*MSkin), list(MSkin*Nr_atoms), tmp(Nr_atoms), cnt, cct +real(dp) :: Lx,Ly,Lz,Tx,Ty,Tz,RR(3),TT(3),dist,dLx,dLy,dLz + +real(dp) :: start, finish + +start = mls() + +N = Nr_atoms +Lx = LBox(1) +Ly = LBox(2) +Lz = LBox(3) ! Dimensions of periodic BC +nx = floor(Lx/Rcut) +ny = floor(Ly/Rcut) +nz = floor(Lz/Rcut) ! Division into # cell boxes: nx, ny, nz +Rx(1:Nr_atoms) = R_X(1:Nr_atoms) !+Lx +Ry(1:Nr_atoms) = R_Y(1:Nr_atoms) !+Ly +Rz(1:Nr_atoms) = R_Z(1:Nr_atoms) !+Lz + +if ((min(nx,ny,nz).lt.3).or.(Nr_atoms.lt.80)) then + +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(i,j,k,l,m,RR,TT,Tx,Ty,Tz,dist,tmp,cnt) + do i = 1,N + cnt = 0 + tmp = ZERO + RR(1) = Rx(i) + RR(2) = Ry(i) + RR(3) = Rz(i) + do m = 1,N + do j = -1,1 + do k = -1,1 + do l = -1,1 + Tx = Rx(m)+j*Lx ! Search all neigbors within a single translation (multiple translations could be necessary for small systems! + Ty = Ry(m)+k*Ly + Tz = Rz(m)+l*Lz + TT(1) = Tx + TT(2) = Ty + TT(3) = Tz + dist = norm2(RR-TT) + !if ((dist.lt.Rcut).and.(dist.gt.1e-12)) then ! Neighbors within Rcut inlcuidng translated atoms in the "skin" + if ((dist < Rcut)) then + cnt = cnt + 1 + nndist(cnt,i) = dist + nnRx(cnt,i) = Tx + nnRy(cnt,i) = Ty + nnRz(cnt,i) = Tz + nnType(cnt,i) = m ! Neigbor is number of original ordering number m in the box that might have been stranslated to the skin + tmp(m) = m + endif + enddo + enddo + enddo + enddo + nrnnlist(i) = cnt + enddo +!$OMP END PARALLEL DO + +else + + head = ZERO ! Linked list that keeps track of all atoms in the nx*ny*nz small boxes + list = ZERO ! + do i = 1,N + cell = 1 + floor(nx*Rx(i)/Lx) + floor(ny*Ry(i)/Ly)*nx + floor(nz*Rz(i)/Lz)*nx*ny + list(i) = head(cell) + head(cell) = i + type(i) = i + enddo + + !%%% And now add a skin or surface buffer to account for periodic BC, all 26 of them! + cnt = 0 + do i = 1,nx*ny ! All boxes in the first (z=0) layer + t = head(i) + do while (t.gt.0) ! and all atoms of this first layer + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)+Lz ! Add skin atoms with coordinates translated by Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = 1,nx*ny*nz,nx ! All boxes in another (x=0) layer + t = head(i) + do while (t.gt.0) ! and all atoms in that layer + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx ! Add skin atoms with coordinates translated by Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = 1,nx*ny*nz,nx*ny ! Continue ... + do k = i,i+nx-1 + t = head(k) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + enddo + cct = 0 + do i = nx*ny*(nz-1)+1,nx*ny*nz + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + cct = cct + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx,nx*ny*nz,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*(ny-1)+1,nx*ny*nz,nx*ny + do k = i,i+nx-1 + t = head(k) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + enddo + do i = 1,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*(ny-1)+1,nx*(ny-1)+nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*ny*(nz-1)+1,nx*ny*(nz-1)+nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*ny*(nz-1)+nx*(ny-1)+1,nx*ny*(nz-1)+nx*(ny-1)+nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t) + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = 1,nx*ny,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*ny*(nz-1)+1,nx*ny*nz,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*ny*(nz-1)+nx,nx*ny*nz,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx,nx*ny,nx + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t) + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = 1,nx*ny*nz,nx*ny + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*(ny-1)+1,nx*ny*nz,nx*ny + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx*(ny-1)+nx,nx*ny*nz,nx*ny + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + do i = nx,nx*ny*nz,nx*ny + t = head(i) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t) + type(N+cnt) = t + t = list(t) + enddo + enddo + t = head(1) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*(ny-1)+1) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*(ny-1)+nx) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)+Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*ny*(nz-1)+1) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*ny*(nz-1)+nx) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)+Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*ny*(nz-1)+nx*(ny-1)+1) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)+Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + t = head(nx*ny*nz) + do while (t.gt.0) + cnt = cnt + 1 + Rx(N+cnt) = Rx(t)-Lx + Ry(N+cnt) = Ry(t)-Ly + Rz(N+cnt) = Rz(t)-Lz + type(N+cnt) = t + t = list(t) + enddo + Nskin = cnt + + !! And now create a list for everything including the Skin/buffer layer +! [max(Rz),min(Rz)] + dLx = Lx/nx + dLy = Ly/ny + dLz = Lz/nz + Rx = Rx+dLx + Ry = Ry+dLy + Rz = Rz+dLz ! Shift to avoid negative coordinates + Lx = Lx + 2*dLx + Ly = Ly + 2*dLy + Lz = Lz + 2*dLz + nx = nx+2 + ny = ny+2 + nz = nz+2 + head = ZERO + list = ZERO + do i = 1,N+Nskin + cell = 1 + floor(nx*Rx(i)/Lx) + floor(ny*Ry(i)/Ly)*nx + floor(nz*Rz(i)/Lz)*nx*ny + list(i) = head(cell) + head(cell) = i + enddo + +!! OPENMP LOOP, JUST LIKE IN CHRISTIAN'S CODE, ESSENTIALLY ALL TIME AND WORK IS SPENT HERE + do i = 1,N ! Go through all atoms + cnt = 0 + do j = -1,1 ! Translate position to neighboring small boxes + do k = -1,1 + do l = -1,1 + Tx = Rx(i)+j*dLx + Ty = Ry(i)+k*dLy + Tz = Rz(i)+l*dLz + cell = 1 + floor(nx*Tx/Lx) + floor(ny*Ty/Ly)*nx + floor(nz*Tz/Lz)*nx*ny ! and extract all atoms in those small + t = head(cell) ! neighbor boxes ... + do while (t.gt.0) + RR(1) = Rx(i) - Rx(t) + RR(2) = Ry(i) - Ry(t) + RR(3) = Rz(i) - Rz(t) + dist = norm2(RR) + !if (dist.lt.Rcut) then ! All atoms including the skin within Rcut WITH ITSELF! + if (dist.lt.Rcut .and. dist .gt. 1d-12) then ! All atoms including the skin within Rcut WITHOUT ITSELF! + cnt = cnt + 1 + nndist(cnt,i) = dist + nnRx(cnt,i) = Rx(t)-dLx ! Coordinates without the shift + nnRy(cnt,i) = Ry(t)-dLy + nnRz(cnt,i) = Rz(t)-dLz + nnType(cnt,i) = type(t) + endif + t = list(t) + enddo + enddo + enddo + enddo + nrnnlist(i) = cnt + enddo + finish = mls() +print '("Time for last nn_list = ",f6.3," seconds.")',finish-start +endif + +end subroutine gpmd_nearestneighborlist + + +end module gpmdcov_neighbor_mod diff --git a/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 b/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 new file mode 100644 index 00000000..896074a6 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 @@ -0,0 +1,114 @@ +module gpmdcov_nonequilibrium_mod + + use bml + use latteparser_latte_mod + use gpmdcov_vars + use prg_quantumdynamics_mod + + public :: gpmdcov_get_currents + + real(dp),parameter :: hbar = 0.65821188926_dp + +contains + + !> Computing currents beween atoms + !! \brief This will compute the commutator between the Hamiltonian and the + !! Desity matrix. It will also multiply by a factor -1/hbar to get + !! units of currents. The routine will the compute the effective + !! current that would circulate between each bond given the new + !! Hamiltonian at time t and the previus Density matrix at time t - 1 + !! \param h_bml The hamiltonian of the core+halo subsystem + !! \param p_bml The density matrix for the core+halo ar previous time + !! \param z_bml The overlap matriz at time t + !! \param subsyhindex The hindex of the subsystem. subsyhindex(1,i) = + !! Initial orbital index for atom i + !! nats_core Number of atoms of the core subsystem + !! norb_core Number of orbitals for the core subsystem + !! core_halo_index Indices that maps subsystem to full system. atom "i" in the + !! subsystem will have "core_halo_index(i) + 1" index in the full system + !! symbols Symbols list for the full system + !! currthr Threshold to plot the current + !! + subroutine gpmdcov_get_currents(h_bml,p_bml,z_bml,subsyhindex,nats_core,& + &norb_core,core_halo_index,symbols,currthr) + + implicit none + + character(2), allocatable :: symbols(:) + integer :: ati, atj, ii, jj + integer :: nats_core, norbH, norbP, norb_core + integer, allocatable :: core_halo_index(:) + integer, allocatable, intent(in) :: subsyhindex(:,:) + real(dp) :: currthr, factor + real(dp), allocatable :: aux_dense(:,:), com_dense(:,:), currents(:,:) + real(dp), allocatable :: hCore_dense(:,:), h_dense(:,:), pCore_dense(:,:), p_dense(:,:) + real(dp), allocatable :: sinv_dense(:,:), zCore_dense(:,:), z_dense(:,:) + type(bml_matrix_t) :: h_bml, p_bml, zCore_bml, z_bml + + + !Since p is from step MDstep - 1 it could have a + !different number of orbitals compared to h + norbH = bml_get_N(h_bml) + norbP = bml_get_N(p_bml) + + allocate(h_dense(norbH,norbH)) + allocate(p_dense(norbP,norbP)) + allocate(z_dense(norbH,norbH)) + call bml_export_to_dense(h_bml,h_dense) + call bml_export_to_dense(p_bml,p_dense) + call bml_export_to_dense(z_bml,z_dense) + + allocate(hCore_dense(norb_core,norb_core)) + hCore_dense(1:norb_core,1:norb_core) = h_dense(1:norb_core,1:norb_core) + deallocate(h_dense) + + allocate(pCore_dense(norb_core,norb_core)) + pCore_dense(1:norb_core,1:norb_core) = p_dense(1:norb_core,1:norb_core) + deallocate(p_dense) + + allocate(zCore_dense(norb_core,norb_core)) + zCore_dense(1:norb_core,1:norb_core) = z_dense(1:norb_core,1:norb_core) + deallocate(z_dense) + + allocate(com_dense(norb_core,norb_core)) + allocate(aux_dense(norb_core,norb_core)) + + factor = -1.0_dp/hbar + + sinv_dense = matmul(zCore_dense,zCore_dense) + com_dense = matmul(hCore_dense,pCore_dense) + com_dense = matmul(sinv_dense,com_dense) + aux_dense = matmul(pCore_dense,hCore_dense) + com_dense = com_dense - matmul(aux_dense,sinv_dense) + allocate(currents(nats_core,nats_core)) + + currents = 0.0_dp + do ati = 1,nats_core-1 + do atj = ati+1,nats_core + currents(ati,atj)=0.0_dp + do ii = subsyhindex(1,ati),subsyhindex(2,ati) + do jj = subsyhindex(1,atj),subsyhindex(2,atj) + currents(ati,atj) = currents(ati,atj) + com_dense(ii,jj) + enddo + enddo + enddo + enddo + + do ii=1,nats_core + do jj=ii+1,nats_core + ati = core_halo_index(ii)+1 + atj = core_halo_index(jj)+1 + if(abs(currents(ii,jj)) .gt. currthr)then + write(*,*)"Currents:",MDstep,ati,atj,symbols(ati),symbols(atj),& + &Currents(ii,jj) + endif + enddo + enddo + + deallocate(aux_dense, com_dense, currents) + deallocate(hCore_dense, pCore_dense) + deallocate(sinv_dense, zCore_dense) + + end subroutine gpmdcov_get_currents + +end module gpmdcov_nonequilibrium_mod diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 new file mode 100644 index 00000000..cdba1426 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -0,0 +1,299 @@ +!> Module for parsing related functions. +!! \brief This module will be used to have parsing routines. +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_parser_mod + + use prg_kernelparser_mod + + integer, parameter :: dp = kind(1.0d0) + + !> gpmd type + type, public :: gpmd_type + + !> Job name. + character(len=100) :: job_name + + !> Output file name + character(len=20) :: output_file_name + + !> Optimize with dumped MD + logical :: dovelresc + + !> Initial temperature + real(dp) :: temp0 + + !> Velocity rescaling factor + real(dp) :: velresc_fact + + !> SMD Force Constant for Start of Switch + real(dp) :: smdforceconststart + + !> SMD Force Constant for End of Switch + real(dp) :: smdforceconstend + + !> SMD optimal separation distance + real(dp) :: smdr0 + + !> SMD minimum separation distance + real(dp) :: smdminsep + + !> SMD maximum separation distance + real(dp) :: smdmaxsep + + !Output control + logical :: writetraj + + !Restart file control + logical :: writerestart + + !Write residue in trajectory + logical :: writeResTraj + + !Restart from dump file + logical :: restartfromdump + + !> Write trajectory coordinates each + integer :: writetreach + + !> Write trajectory coordinates each + integer :: writerestarteach + + !> Replicae system in x direction + integer :: replicatex + + !> Replicae system in y direction + integer :: replicatey + + !> Replicae system in z direction + integer :: replicatez + + !> Track the rectivity + logical :: trackreactivity + + !> Track parts from partition. If 0, no part is tracked + integer :: tracknparts + + !> Track parts from partition + integer, allocatable :: trackparts(:) + + !> Dump into a file each dumpeach steps + integer :: dumpeach + + !> Number of steered pairs for SMD + integer :: smdnumpairs + + !> SMD atom 1 indicies + integer, allocatable :: smdatomind1(:) + + !> SMD atom 2 indicies + integer, allocatable :: smdatomind2(:) + + !> Use LATTE code to compute Hamiltonians + logical :: useLatte + + !> Use deuterium mass for H atoms + logical :: htod + + !> Use Steered MD (SMD) + logical :: usesmd + + !> Output trajectory format + character(len=100) :: traj_format + + !> Use Langevin integration + logical :: langevin + + !> Langevin method + character(len=100) :: langevin_method + + !> Langevin decay constant + real(dp) :: langevin_gamma + + !> Number of initial minimization steps before dynamics + integer :: minimization_steps + + !> Compute currents option + logical :: compcurr + + !> Currents Threshold + real(dp) :: currthr + + !> Translate and fold to box + logical :: trfl + + !> Use Vectorized SKBlock method in PROGRESS + logical :: usevectsk + + + end type gpmd_type + + private + + public :: gpmdcov_parse + +contains + + !> Gpmdcov parser. + !! \brief This module is used to parse all the input variables for this program. + !! Adding a new input keyword to the parser: + !! - If the variable is real, we have to increase nkey_re. + !! - Add the keyword (character type) in the keyvector_re vector. + !! - Add a default value (real type) in the valvector_re. + !! - Define a new variable and pass the value through valvector_re(num) + !! where num is the position of the new keyword in the vector. + !! \param filename File name for the input. + !! \param gpmd type. + !! + subroutine gpmdcov_parse(filename,gpmdt) + implicit none + character(len=*), intent(in) :: filename + type(gpmd_type), intent(inout) :: gpmdt + integer, parameter :: nkey_char = 4, nkey_int = 9, nkey_re = 7, nkey_log = 12 + integer :: i + character(20) :: dummyc + logical :: inGPMD + + !Library of keywords with the respective defaults. + character(len=50), parameter :: keyvector_char(nkey_char) = [character(len=50) :: & + & 'JobName=', 'Var2C=', 'TrajectoryFormat=', 'LangevinMethod='] + character(len=100) :: valvector_char(nkey_char) = [character(len=100) :: & + &'MyMol', 'Def2', 'PDB', 'Goga'] + + character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & + & 'WriteCoordsEach=',"Var2I=","ReplicateX=","ReplicateY=","ReplicateZ=","PartsToTrack=",& + & "DumpEach=","MinimizationSteps=","SMDNumPairs="] + integer :: valvector_int(nkey_int) = (/ & + & 1, 1, 0, 0, 0, 0, 0, 0, 0/) + + character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & + & 'VRFactor=','InitialTemperature=','LangevinGamma=','SMDForceConstantStart=',& + & 'SMDForceConstantEnd=','SMDR0=','CurrentThreshold='] + real(dp) :: valvector_re(nkey_re) = (/& + & 0.0_dp, 0.0_dp, 0.01_dp, 0.0_dp,0.2_dp,2.0_dp,0.1_dp/) + + character(len=50), parameter :: keyvector_log(nkey_log) = [character(len=50) :: & + &'DoVelocityRescale=','WriteResidueInTrajectory=','WriteTrajectory=','TrackReactivity=',& + &'RestartFromDump=','UseLATTE=','HtoD=','LangevinDynamics=','UseSMD=', & + &'ComputeCurrents=', 'TranslateAndFoldToBox=', 'UseVectSKBlock='] + logical :: valvector_log(nkey_log) = (/& + &.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false., & + &.false.,.True.,.false./) + + !Start and stop characters + character(len=50), parameter :: startstop(2) = [character(len=50) :: & + 'GPMD{', '}'] + + call prg_parsing_kernel(keyvector_char,valvector_char& + ,keyvector_int,valvector_int,keyvector_re,valvector_re,& + keyvector_log,valvector_log,trim(filename),startstop) + + !Characters + gpmdt%job_name = valvector_char(1) + gpmdt%traj_format = valvector_char(3) + gpmdt%langevin_method = valvector_char(4) + + !Integer + gpmdt%writetreach = valvector_int(1) + gpmdt%replicatex = valvector_int(3) + gpmdt%replicatey = valvector_int(4) + gpmdt%replicatez = valvector_int(5) + gpmdt%tracknparts = valvector_int(6) + inGPMD = .false. + if(gpmdt%tracknparts > 0)then + write(*,*)"Reading parts to track ..." + open(1, file=trim(filename)) + do i = 1,10000 + read(1,*) dummyc + write(*,*) trim(adjustl(dummyc)) + if(trim(adjustl(dummyc)) == "Parts[")then + exit + end if + if(trim(adjustl(dummyc)) == "GPMD{") inGPMD = .true. + + if(trim(dummyc) == "}" .and. inGPMD)then + write(*,*)'ERROR: No part numbers defined' + write(*,*)"Here is an example block you should add to the" + write(*,*)"input file" + write(*,*)"" + write(*,*)"Parts[" + write(*,*)" 1" + write(*,*)" 100" + write(*,*)"]" + stop + end if + end do + allocate(gpmdt%trackparts(gpmdt%tracknparts)) + do i = 1,gpmdt%tracknparts + read(1,*)gpmdt%trackparts(i) + end do + write(*,*)"" + close(1) + endif + + gpmdt%dumpeach = valvector_int(7) + gpmdt%minimization_steps = valvector_int(8) + gpmdt%smdnumpairs = valvector_int(9) + if(gpmdt%smdnumpairs > 0) then + write(*,*) "Reading SMD pair atom indicies" + open(1, file=trim(filename)) + do i = 1,10000 + read(1,*) dummyc + !write(*,*) trim(adjustl(dummyc)) + if(trim(adjustl(dummyc)) == "SMDPairs[")then + exit + end if + if(trim(adjustl(dummyc)) == "GPMD{") inGPMD = .true. + + if(trim(dummyc) == "}" .and. inGPMD)then + write(*,*)'ERROR: No SMD Pairs defined' + write(*,*)"Here is an example block you should add to the" + write(*,*)"input file" + write(*,*)"" + write(*,*)"SMDPairs[" + write(*,*)" 1 3" + write(*,*)" 100 102" + write(*,*)"]" + stop + end if + end do + allocate(gpmdt%smdatomind1(gpmdt%smdnumpairs)) + allocate(gpmdt%smdatomind2(gpmdt%smdnumpairs)) + do i = 1,gpmdt%smdnumpairs + read(1,*) gpmdt%smdatomind1(i), gpmdt%smdatomind2(i) + end do + write(*,*)"" + close(1) + endif + do i = 1,gpmdt%smdnumpairs + write(*,*) "SMD Pairs Atom 1 ",gpmdt%smdatomind1(i)," SMD Pairs Atom 2 ",gpmdt%smdatomind2(i) + enddo + + !Reals + gpmdt%velresc_fact = valvector_re(1) + gpmdt%temp0 = valvector_re(2) + gpmdt%langevin_gamma = valvector_re(3) + gpmdt%smdforceconststart = valvector_re(4) + gpmdt%smdforceconstend = valvector_re(5) + gpmdt%smdr0 = valvector_re(6) + gpmdt%currthr = valvector_re(7) + + !Logs + gpmdt%dovelresc = valvector_log(1) + gpmdt%writeResTraj = valvector_log(2) + gpmdt%writeTraj = valvector_log(3) + gpmdt%trackreactivity = valvector_log(4) + gpmdt%restartfromdump = valvector_log(5) + gpmdt%useLatte = valvector_log(6) + gpmdt%htod = valvector_log(7) + gpmdt%langevin = valvector_log(8) + gpmdt%usesmd = valvector_log(9) + gpmdt%compcurr = valvector_log(10) + gpmdt%trfl = valvector_log(11) + gpmdt%usevectsk = valvector_log(12) + + end subroutine gpmdcov_parse + +end module gpmdcov_parser_mod diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 new file mode 100644 index 00000000..ee8b0579 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -0,0 +1,280 @@ +module gpmdcov_Part_mod + +contains + + !> Partition by systems + !! + subroutine gpmdcov_Part(ipreMD) + + use gpmdcov_vars + use gpmdcov_reshuffle_mod + use gpmdcov_writeout_mod + use gpmdcov_allocation_mod + + integer, allocatable :: graph_h(:,:) + integer, allocatable :: graph_p(:,:) + real(dp) :: mls_ii + real, allocatable :: onesMat(:,:) + integer :: iipt,ipreMD + integer :: maxCoreHalo, minCoreHalo, averageCoreHalo + integer :: maxCoreHaloLoc, maxCoreHaloRank + integer :: coreHaloP1, coreP1 + integer :: myMdim + + if(gsp2%mdim < 0)then + myMdim = sy%nats + elseif(gsp2%mdim > sy%nats)then + myMdim = sy%nats + else + myMdim = gsp2%mdim + endif + + if(ipreMD == 1)then + call gpmdcov_msMem("gpmdcov_Part", "Before prg_get_covgraph",lt%verbose,myRank) + call prg_get_covgraph(sy,nl%nnStruct,nl%nrnnstruct& + ,gsp2%bml_type,gsp2%covgfact,g_bml,myMdim,lt%verbose) + call gpmdcov_msMem("gpmdcov_Part", "After prg_get_covgraph",lt%verbose,myRank) + else + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !Anders' way of graph construction. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if(gpat%TotalParts > 1)then + call gpmdcov_msI("gpmdcov_Part","In prg_get_covgraph_h ...",lt%verbose,myRank) + mls_ii = mls() + call prg_get_covgraph_h(sy,nl%nnStruct,nl%nrnnstruct,gsp2%nlgcut,graph_h,myMdim,lt%verbose) + call gpmdcov_msIII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iipt=1,partsInEachRank(myRank) + ipt= reshuffle(iipt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + + call prg_collect_graph_p(syprt(ipt)%estr%orho,gpat%sgraph(ipt)%llsize,sy%nats,syprt(ipt)%estr%hindex,& + gpat%sgraph(ipt)%core_halo_index,graph_p,gsp2%gthreshold,myMdim,lt%verbose) + + call bml_deallocate(syprt(ipt)%estr%orho) + + enddo + + mls_i = mls() + +! call gpmdcov_mat2VectInt(graph_p,auxVectInt,sy%nats,myMdim) + +#ifdef DO_MPI + if (getNRanks() > 1) then + call prg_sumIntReduceN(graph_p, myMdim*sy%nats) + ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) + endif +#endif + + ! call gpmdcov_vect2MatInt(auxVectInt,graph_p,sy%nats,myMdim) + ! deallocate(auxVectInt) +! write(*,*)graph_p + call gpmdcov_msIII("gpmdcov_Part","Time for prg_sumIntReduceN for graph "//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + + call gpmdcov_msI("gpmdcov_Part","In prg_merge_graph ...",lt%verbose,myRank) + mls_ii = mls() + call prg_wait() + call prg_merge_graph(graph_p,graph_h) + call prg_wait() + call gpmdcov_msIII("gpmdcov_Part","Time for prg_merge_graph "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + + !call prg_wait() + !call prg_wait() + + !Transform graph into bml format. + ! if(mod(mdstep,gsp2%parteach)==0 .or. mdstep <= 1)then !Only for debug purposes (Halos are updated every step) + if(bml_allocated(g_bml)) call bml_deallocate(g_bml) + !call bml_zero_matrix(gsp2%bml_type,bml_element_real,kind(1.0),sy%nats,myMdim,g_bml,lt%bml_dmode) + call bml_zero_matrix(gsp2%bml_type,bml_element_real,kind(1.0),sy%nats,myMdim,g_bml) + !call bml_zero_matrix(gsp2%bml_type,bml_element_real,dp,sy%nats,myMdim,g_bml) + + call gpmdcov_msMem("gpmdcov_Part","Before prg_graph2bml",lt%verbose,myRank) + mls_ii = mls() + call prg_graph2bml(graph_p,gsp2%bml_type,g_bml) +! if(lt%verbose == 7 .and. myRank == 1)then +! call bml_write_matrix(g_bml,"g_bml.mtx") +! stop +! endif + + + if(allocated(graph_p)) deallocate(graph_p) + if(allocated(graph_h)) deallocate(graph_h) + + + call gpmdcov_msMem("gpmdcov_Part","After prg_graph2bml",lt%verbose,myRank) + call gpmdcov_msIII("gpmdcov_Part","Time for prg_graph2bml "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + !endif + else + allocate(onesMat(sy%nats,sy%nats)) + onesMat = 1.0 + call bml_zero_matrix(gsp2%bml_type,bml_element_real,kind(1.0),sy%nats,myMdim,g_bml) + call bml_import_from_dense(gsp2%bml_type, onesMat, g_bml, 0.0_dp, sy%nats) + deallocate(onesMat) + endif + + endif + + if(allocated(syprt))then + do ipt=1,gpat%TotalParts + call prg_destroy_estr(syprt(ipt)%estr) + enddo + + do ipt=1,gpat%TotalParts + call prg_destroy_subsystems(syprt(ipt),lt%verbose) + enddo + deallocate(syprt) + endif + + if (myRank == 1 .and. lt%verbose >= 5) then + call bml_print_matrix("gcov",g_bml,0,15,0,15) + endif + + if(mod(mdstep,gsp2%parteach)==0 .or. mdstep <= 1)then + if(lt%verbose >= 1 .and. myRank == 1)write(*,*)"In graph_part .." + mls_ii = mls() + call gpmd_graphpart() + firstKernel = .true. + if(mdstep >= 1) newPart = .true. + if(lt%verbose >= 3 .and. myRank == 1)write(*,*)"Time for gpmd_graphpart "//to_string(mls()-mls_ii)//" ms" + endif + + !> \todo have the preconditioner construction independent of the partitioning + !if(mod(mdstep,10)==0 .or. mdstep <= 1)then + ! if(lt%verbose >= 1 .and. myRank == 1)write(*,*)"In graph_part .." + ! firstKernel = .true. + !endif + + + + !To partition by molecule. + ! write(*,*)"part by mol" + ! call prg_molpartition(sy,nparts_cov,nl%nnStructMindist,nl%nnStruct,nl%nrnnstruct,"O ",gpat) + +#ifdef SANITY_CHECK + write(*, *) "sanity check before bml_matrix2submatrix_index" + do ipt = 1,gpat%TotalParts + do iipt = ipt+1,gpat%TotalParts + do i = 1, gpat%sgraph(ipt)%llsize + do j = 1, gpat%sgraph(iipt)%llsize + if(gpat%sgraph(ipt)%core_halo_index(i) == gpat%sgraph(iipt)%core_halo_index(j))then + write(*,*)"cores are repeated in partitions",mdstep + write(*,*)ipt,gpat%sgraph(ipt)%core_halo_index(i),iipt,gpat%sgraph(ipt)%core_halo_index(j) + write(*,*)i,j + stop + endif + enddo + enddo + enddo + enddo +#endif + + mls_ii = mls() + do i=1,gpat%TotalParts + call bml_matrix2submatrix_index(g_bml,& + gpat%sgraph(i)%nodeInPart,gpat%nnodesInPart(i),& + gpat%sgraph(i)%core_halo_index, & + vsize,.true.) + gpat%sgraph(i)%lsize = vsize(1) + gpat%sgraph(i)%llsize = vsize(2) + if(myRank == 1 .and. lt%verbose == 3) write(*,*)"part",i,"cores, cores+halo",vsize(2),vsize(1) + enddo + + if(myRank == 1)then + maxCoreHalo = maxval(gpat%sgraph(:)%lsize) + maxCoreHaloLoc = maxloc(gpat%sgraph(:)%lsize,dim=1) + maxCoreHaloRank = 0 + do i = 1,gpat%totalProcs + if (maxCoreHaloLoc.ge.gpat%localPartMin(i).and.maxCoreHaloLoc.le.gpat%localPartMax(i)) then + maxCoreHaloRank = i + endif + enddo + minCoreHalo = minval(gpat%sgraph(:)%lsize) + if(gpat%TotalParts.eq.0)then + averageCoreHalo = sum(gpat%sgraph(:)%lsize) + else + averageCoreHalo = sum(gpat%sgraph(:)%lsize)/gpat%TotalParts + endif + call gpmdcov_msI("gpmdcov_Part","Max and min core+halo "//to_string(maxCoreHalo)//" "//& + &to_string(minCoreHalo),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_Part","Max core+halo in (part rank) "//to_string(maxCoreHaloLoc)//" "//& + &to_string(maxCoreHaloRank),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_Part","Average core+halo "//to_string(averageCoreHalo)//" ",1,myRank) + if(gpmdt%tracknparts > 0)then + do i = 1,gpmdt%tracknparts + coreHaloP1 = gpat%sgraph(gpmdt%trackparts(i))%lsize + coreP1 = gpat%sgraph(gpmdt%trackparts(i))%llsize + call gpmdcov_msI("gpmdcov_Part","Part "//to_string(gpmdt%trackparts(i))//& + &" core+halo "//to_string(coreHaloP1)//" ",1,myRank) + call gpmdcov_msI("gpmdcov_Part","Part "//to_string(gpmdt%trackparts(i))//& + &" core "//to_string(coreP1)//" ",1,myRank) + enddo + endif + endif + + call gpmdcov_msIII("gpmdcov_Part","Time for bml_matrix2submatrix_index "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + + call gpmdcov_reshuffle() + +#ifdef SANITY_CHECK + write(*, *) "sanity check after bml_matrix2submatrix_index" + do ipt = 1,gpat%TotalParts + do iipt = ipt+1,gpat%TotalParts + do i = 1, gpat%sgraph(ipt)%llsize + do j = 1, gpat%sgraph(iipt)%llsize + if(gpat%sgraph(ipt)%core_halo_index(i) == gpat%sgraph(iipt)%core_halo_index(j))then + write(*,*)"cores are repeated in partitions",mdstep + write(*,*)ipt,gpat%sgraph(ipt)%core_halo_index(i),iipt,gpat%sgraph(ipt)%core_halo_index(j) + write(*,*)i,j + stop + endif + enddo + enddo + enddo + enddo +#endif + + if(allocated(syprt))deallocate(syprt) + allocate(syprt(gpat%TotalParts)) + !For currents + if(gpmdt%compcurr)then + if(MDStep == 0 .or. mod(mdstep,gsp2%parteach)==0)then + if(allocated(oldsyprt))deallocate(oldsyprt) + allocate(oldsyprt(gpat%TotalParts)) + endif + endif + + !> For every partition get the partial CH systems. + call gpmdcov_msI("gpmdcov_Part","Getting CH subsystems ...",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov_Part","Before prg_get_subsystem",lt%verbose,myRank) + mls_ii = mls() + +#ifdef DO_MPI + !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iipt=1,partsInEachRank(myRank) + ipt= reshuffle(iipt,myRank) +#else + do ipt = 1,gpat%TotalParts +#endif + + call prg_get_subsystem(sy,gpat%sgraph(ipt)%lsize,gpat%sgraph(ipt)%core_halo_index,syprt(ipt)) + enddo + + call gpmdcov_msIII("gpmdcov_Part","Time for prg_get_subsystem "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + call gpmdcov_msMem("gpmdcov","After prg_get_subsystem",lt%verbose,myRank) + + !To analyze partitions with VMD. + if(lt%verbose >= 4)then + if(mod(mdstep,gsp2%parteach) == 0 .or. mdstep == 1)then + call gpmdcov_writepartitionout(sy,syprt,gpat,reshuffle,partsInEachRank,myRank) + endif + endif + + + end subroutine gpmdcov_Part + +end module gpmdcov_Part_mod diff --git a/examples/gpmdk/src/gpmdcov_preparemd.F90 b/examples/gpmdk/src/gpmdcov_preparemd.F90 new file mode 100644 index 00000000..7edede69 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_preparemd.F90 @@ -0,0 +1,95 @@ +module gpmdcov_prepareMD_mod + +contains + + !> Preparing for MD + !! + subroutine gpmdcov_prepareMD(temp0) + use gpmdcov_vars + use gpmdcov_writeout_mod + Implicit none + real(dp), intent(in) :: temp0 + + call gpmdcov_msI("gpmdcov_prepareMD","In gpmdcov_prepareMD...",lt%verbose,myRank) + + !> Initialize velocities + if(.not.allocated(sy%velocity))then + allocate(sy%velocity(3,sy%nats)) + sy%velocity(1,:) = 0.0_dp; sy%velocity(2,:) = 0.0_dp; sy%velocity(3,:) = 0.0_dp; ! Initialize velocities + endif + + !> Kinetic energy in eV (MVV2KE: unit conversion) + MVV2KE = 166.0538782_dp/1.602176487_dp + + KE2T = 1.0_dp/0.000086173435_dp + + !> Pressure in bar from eV / Angstrom^3 + + EVOVERV2P = 1.602176487_dp*1000.0_dp + + if(temp0 > 1.0E-10)then + if(.not.gpmdt%restartfromdump) call gpmdcov_addVelocity(temp0,sy%velocity,sy%mass) + endif + + end subroutine gpmdcov_prepareMD + + !> Adding random velocity + !! + subroutine gpmdcov_addVelocity(temp0,velocity,mass) + use gpmdcov_vars + implicit none + real(dp), intent(inout) :: velocity(:,:) + real(dp), intent(in) :: mass(:) + real(dp), intent(in) :: temp0 + real(dp) :: kinE,iVel,ran + integer :: nats,myi,myseed,ssize + integer, allocatable :: seedin(:) + + nats = size(velocity,dim=2) + + kinE = temp0 * (1.5_dp)*real(nats,dp)/KE2T + kinE = kinE/(0.5_dp*MVV2KE) + kinE = kinE/real(nats,dp) + + myseed = 12345 + call random_seed() + call random_seed(size=ssize) + allocate(seedin(ssize)) + seedin = myseed + call random_seed(PUT=seedin) + + !call random_seed(myseed) + + !> Distribute kin E + do myi = 1,nats + iVel = sqrt(kinE/mass(myi)) + call random_number(ran) + write(*,*) + velocity(1,myi) = (2.0_dp*ran - 1.0_dp) + call random_number(ran) + velocity(2,myi) = (2.0_dp*ran - 1.0_dp) + call random_number(ran) + velocity(3,myi) = (2.0_dp*ran - 1.0_dp) + + velocity(:,myi) = velocity(:,myi)/(norm2(velocity(:,myi))) + velocity(:,myi) = iVel*sy%velocity(:,myi) + enddo + +#ifdef DO_MPI + if (numRanks .gt. 1) then !THIS IS VERY IMPORTANT + + call prg_sumRealReduceN(velocity(1,:), nats) + call prg_sumRealReduceN(velocity(2,:), nats) + call prg_sumRealReduceN(velocity(3,:), nats) + + velocity = velocity/real(numRanks,dp) + endif +#endif + + + + end subroutine gpmdcov_addVelocity + + + +end module gpmdcov_prepareMD_mod diff --git a/examples/gpmdk/src/gpmdcov_rankN.F90 b/examples/gpmdk/src/gpmdcov_rankN.F90 new file mode 100644 index 00000000..6522717e --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_rankN.F90 @@ -0,0 +1,38 @@ +r = K0*(q[n]_core-n_core) +do i = 1, Nranks + + do l = 1, nparts + + ! Compute a part for the preconditioner residual vector + ! K0Res_core = K0_core*r + + enddo + + ! v_i = r/norm(r) + ! v_i orthonormalized to all v_j, j = 1,2,..., i-1 + ! v_i (charge perturbation vector) -> VCoulomb + + do l = 1, nparts + + ! VCoulomb for core + halo of each subgraph + ! DMPRT(H0,H1 = VCoulomb, ....) -> P1' (from part), + ! dPmu = beta*P0*(I-P0) (from part), + ! Core traces of P1', and dPmu, i.e. TrP1' and TrdPmu (fore each + ! core) + + enddo + + ! mu_1 = -Sum TrP1' / Sum TrPmu over all graphs + + do l = 1, nparts + + ! P1 = P1' + mu_1*dPmu (core + halo) + ! Trace(P1*S) for each atom and core -> dq[n+lambda*v_i]/dlambda + + enddo + + ! r = dq[n+lambda*v_i]/d_lambda - v_i + ! r = K0*r + ! f_vi = r + +enddo diff --git a/examples/gpmdk/src/gpmdcov_reshuffle.F90 b/examples/gpmdk/src/gpmdcov_reshuffle.F90 new file mode 100644 index 00000000..649a73cd --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_reshuffle.F90 @@ -0,0 +1,73 @@ +module gpmdcov_reshuffle_mod + + contains + + !> Reshuffle the parts + !! partsInEachRank(getNRanks()) stores the number of partitions assigned to rank i + !! reshuffle(j,i) assigns partition reshuffle(j,i) to rank i for j=1,partsInEachRank(getNRanks()) + !! + subroutine gpmdcov_reshuffle() + use gpmdcov_vars + use gpmdcov_writeout_mod + + integer :: maxnparts, np + + maxnparts = 0 + do i=1,numRanks + np = gpat%localPartMax(i)-gpat%localPartMin(i)+1 + maxnparts = max(maxnparts,np) + enddo + + if(allocated(reshuffle))then + deallocate(reshuffle) + deallocate(partsInEachRank) + endif + + allocate(reshuffle(maxnparts,numRanks)) + allocate(partsInEachRank(numRanks)) + + reshuffle = 0 + icount = 0 + partsInEachRank = 0 + + do j=1,maxnparts + do i=1,getNRanks() + np = gpat%localPartMax(i)-gpat%localPartMin(i)+1 + if(np > partsInEachRank(i))then + icount = icount + 1 + partsInEachRank(i) = partsInEachRank(i) + 1 + reshuffle(partsInEachRank(i),i) = icount + if(icount == nparts) exit + endif + enddo + if(icount == nparts) exit + do i=getNRanks(),1,-1 + np = gpat%localPartMax(i)-gpat%localPartMin(i)+1 + if(np > partsInEachRank(i))then + icount = icount + 1 + partsInEachRank(i) = partsInEachRank(i) + 1 + reshuffle(partsInEachRank(i),i) = icount + if(icount == nparts) exit + endif + enddo + if(icount == nparts) exit + enddo + + costperrankmax = 0.0d0 + costperrankmin = 1.0d+10 + + do i=1,getNRanks() + costperrank = 0.0d0 + do j=1,maxnparts + !if(reshuffle(j,i)>0)write(*,*)i,j,reshuffle(j,i),gpat%sgraph(reshuffle(j,i))%lsize + costperrank = costperrank + real((gpat%sgraph(reshuffle(j,i))%lsize)**3) + enddo + costperrankmax = max(costperrank,costperrankmax) + costperrankmin = min(costperrank,costperrankmin) + enddo + !call gpmdcov_msII("gpmdcov_reshuffle","Measure of workload assymetry per rank & + ! &"//to_string( (costperrankmax - costperrankmin)/costperrankmin ),lt%verbose,myRank) + + end subroutine gpmdcov_reshuffle + +end module gpmdcov_reshuffle_mod diff --git a/examples/gpmdk/src/gpmdcov_restart.F90 b/examples/gpmdk/src/gpmdcov_restart.F90 new file mode 100644 index 00000000..5b955ec6 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_restart.F90 @@ -0,0 +1,57 @@ + !> Dump GPMD + subroutine gpmdcov_restart() + use gpmdcov_vars + + write(*,*)"Restarting ..." + + open(1,file='restart.dmp',form="unformatted",access="sequential",status="old") + read(1)sy%nats + + if(.not.allocated(sy%symbol))allocate(sy%symbol(sy%nats)) + if(.not.allocated(sy%atomic_number))allocate(sy%atomic_number(sy%nats)) + if(.not.allocated(sy%coordinate))allocate(sy%coordinate(3,sy%nats)) + if(.not.allocated(sy%velocity))allocate(sy%velocity(3,sy%nats)) + if(.not.allocated(sy%force))allocate(sy%force(3,sy%nats)) + if(.not.allocated(sy%net_charge))allocate(sy%net_charge(sy%nats)) + if(.not.allocated(sy%mass))allocate(sy%mass(sy%nats)) + if(.not.allocated(sy%spindex))allocate(sy%spindex(sy%nsp)) + if(.not.allocated(sy%lattice_vector))allocate(sy%lattice_vector(3,3)) + if(.not.allocated(sy%spatnum))allocate(sy%spatnum(sy%nsp)) + if(.not.allocated(sy%spmass))allocate(sy%spmass(sy%nsp)) + + read(1)sy%symbol + read(1)sy%atomic_number + read(1)sy%coordinate + read(1)sy%velocity + + read(1)sy%force + read(1)sy%net_charge + read(1)sy%mass + read(1)sy%spindex + read(1)sy%lattice_vector + read(1)sy%spatnum + read(1)sy%spmass + + if(.not.allocated(n))allocate(n(sy%nats)) + if(.not.allocated(n_0))allocate(n_0(sy%nats)) + if(.not.allocated(n_1))allocate(n_1(sy%nats)) + if(.not.allocated(n_2))allocate(n_2(sy%nats)) + if(.not.allocated(n_3))allocate(n_3(sy%nats)) + if(.not.allocated(n_4))allocate(n_4(sy%nats)) + if(.not.allocated(n_5))allocate(n_5(sy%nats)) + + read(1)mdstep + read(1)n + read(1)n_0 + read(1)n_1 + read(1)n_2 + read(1)n_3 + read(1)n_4 + read(1)n_5 + + mdstep = 0 + + close(1) + + end subroutine gpmdcov_restart + diff --git a/examples/gpmdk/src/gpmdcov_rhosolver.F90 b/examples/gpmdk/src/gpmdcov_rhosolver.F90 new file mode 100644 index 00000000..28f3253b --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_rhosolver.F90 @@ -0,0 +1,44 @@ +module gpmdcov_RhoSolver_mod + + contains + !> Solver for computing the density matrix + subroutine gpmdcov_RhoSolver(orthoh_bml,orthop_bml,evects_bml) + use gpmdcov_vars + use gpmdcov_mod + type(bml_matrix_t), intent(in) :: orthoh_bml + type(bml_matrix_t), intent(inout) :: orthop_bml + type(bml_matrix_t), intent(inout) :: evects_bml + + if(lt%verbose >= 1 .and. myRank == 1) write(*,*)"starting solver ..." + + if(lt%method.EQ."GSP2")then + call prg_subgraphSP2Loop(orthoh_bml, g_bml, orthop_bml, gp, lt%threshold) + ! call prg_sp2_alg1_seq(orthoh_bml,orthop_bml,lt%threshold, gp%pp, gp%maxIter, gp%vv) + elseif(lt%method.EQ."SP2")then + call prg_sp2_alg2(orthoh_bml,orthop_bml,lt%threshold, bndfil, sp2%minsp2iter, sp2%maxsp2iter & + ,sp2%sp2conv,sp2%sp2tol,lt%verbose) + elseif(lt%method.EQ."Diag")then + ! call build_density_t0(orthoh_bml,orthop_bml,lt%threshold,bndfil) + ! call prg_build_density_T(orthoh_bml,orthop_bml,lt%threshold,bndfil, 0.1_dp, Ef) + !call prg_build_density_T_Fermi(orthoh_bml,orthop_bml,lt%threshold, 0.1_dp, Ef) + call prg_build_density_T_Fermi(orthoh_bml,orthop_bml,lt%threshold, 0.1_dp, Ef) + if(lt%verbose >= 1 .and. myRank == 1) write(*,*)"ipt =",ipt,"Ef =",Ef + elseif(lt%method.EQ."DiagEf")then + if(bml_get_n(evects_bml) < 0) call bml_deallocate(evects_bml) + call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,evects_bml) + call prg_build_density_T_ed(orthoh_bml,orthop_bml,evects_bml,lt%threshold,bndfil, lt%kbt, Ef, & + & syprt(ipt)%estr%evals, syprt(ipt)%estr%dvals, syprt(ipt)%estr%hindex, gpat%sgraph(ipt)%llsize,lt%verbose) + else + stop "No valid Method in LATTE parameters" + endif + + + + if(lt%verbose >= 2 .and. myRank == 1)then + call bml_print_matrix("orthop_bml",orthop_bml,0,6,0,6) + endif + if(lt%verbose >= 1 .and. myRank == 1) write(*,*)"leaving solver ..." + + end subroutine gpmdcov_RhoSolver + +end module gpmdcov_RhoSolver_mod diff --git a/examples/gpmdk/src/gpmdcov_test.F90 b/examples/gpmdk/src/gpmdcov_test.F90 new file mode 100644 index 00000000..32a93cc4 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_test.F90 @@ -0,0 +1,14 @@ +!> Auxiliary modules. +!! \brief This module will be used to have auxiliary routines. +!! +!! @ingroup GPMDCOV +!! +!! +module gpmdcov_test + +contains + subroutine algo() + + end subroutine algo + +end module gpmdcov_test diff --git a/examples/gpmdk/src/gpmdcov_vars.F90 b/examples/gpmdk/src/gpmdcov_vars.F90 new file mode 100644 index 00000000..9a9a169f --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_vars.F90 @@ -0,0 +1,135 @@ +!> Variables used in the code +!! +module gpmdcov_vars + + use bml + + !Progress and LATTE lib modules. + use prg_progress_mod + use prg_system_mod + use prg_ptable_mod + use latteparser_latte_mod + use huckel_latte_mod + use tbparams_latte_mod + use ham_latte_mod + use coulomb_latte_mod + use prg_charges_mod + use prg_initmatrices_mod + use prg_genz_mod + use prg_nonortho_mod + use prg_pulaymixer_mod + use prg_dos_mod + use prg_densitymatrix_mod + use gpmdcov_neighbor_mod + !use neighborlist_latte_mod + use ppot_latte_mod + use hsderivative_latte_mod + use slaterkosterforce_latte_mod + use prg_pulaycomponent_mod + use nonorthocoulombforces_latte_mod + + ! Graph partitioning modules + use prg_parallel_mod + use prg_timer_mod + use prg_graphsp2parser_mod + use prg_sp2parser_mod + use prg_sp2_mod + use prg_graph_mod + use prg_subgraphLoop_mod + use prg_homolumo_mod + use prg_xlbo_mod + use md_latte_mod + use prg_partition_mod + use prg_extras_mod + + ! gpmdcov modules + use gpmdcov_parser_mod + + implicit none + + integer, parameter :: dp = kind(1.0d0) + integer :: seed = 1 + character(2), allocatable :: TypeA(:,:), TypeB(:,:) + character(3), allocatable :: intKind(:) + character(50) :: inputfile, filename + character(2) :: auxchar + integer :: mdstep, Nr_SCF_It, i, icount, ierr, num + integer :: j, nel, norb, pp(100), nnodes, iii, kk + integer :: nparts, niter=500, npat, ipt, iptt + integer :: ii, jj, iscf, norb_core, totalNorbs + integer :: mdim, shift, nranks, norbsInRank + integer, allocatable :: hindex(:,:), hnode(:), vectorint(:), norbsInEachCHAtRank(:), norbsInEachRank(:) + integer, allocatable :: xadj(:), adjncy(:), CH_count(:), norbsInEachCH(:) + integer, allocatable :: part(:), core_count(:), Halo_count(:,:) + integer, allocatable :: partsInEachRank(:), reshuffle(:,:), npartsVect(:), displ(:), PartsInRankI(:) + real(dp) :: C0, C1, C2, C3 + real(dp) :: C4, C5, ECoul, ECoulU, ECoulK, ECoulR, EKIN, beta, kbt + real(dp) :: EPOT, ERep, Energy, Etot, nocc + real(dp) :: F2V, KE2T, MVV2KE, EVOVERV2P, M_prg_init + real(dp) :: TRRHOH, Temp, Time, alpha + real(dp) :: bndfil, bndfilTotal, cc, coulcut, dt + real(dp) :: dx, egap, egap_glob, ehomo, elumo + real(dp) :: kappa, scferror, traceMult, vv(100) + real(dp) :: sumCubes, maxCH, Ef, smooth_maxCH, pnorm=6 + real(dp) :: dvdw, d, mls_i, Efstep, costperrank, costperrankmax, costperrankmin + real(dp) :: sparsity, entropy + real(dp) :: HOMO, LUMO + real(dp) :: savets + real(dp), allocatable :: eigenvalues(:), evals(:), fvals(:), dvals(:) + real(dp), allocatable :: evalsAll(:), fvalsAll(:), dvalsAll(:) + real(dp), allocatable :: evalsInRank(:), fvalsInRank(:), dvalsInRank(:) + real(dp), allocatable :: GFPUL(:,:), GFSCOUL(:,:), PairForces(:,:) + real(dp), allocatable :: SKForce(:,:), VX(:), VY(:), VZ(:), collectedforce(:,:), smdForce(:,:) + real(dp), allocatable :: charges_old(:), coul_forces(:,:), coul_forces_k(:,:), coul_forces_r(:,:) + real(dp), allocatable :: coul_pot_k(:), coul_pot_r(:), dqin(:,:), dqout(:,:) + real(dp), allocatable :: eigenvals(:), gbnd(:), n(:), n_0(:) + real(dp), allocatable :: n_1(:), n_2(:), n_3(:), n_4(:), acceprat(:) + real(dp), allocatable :: n_5(:), onsitesH(:,:), onsitesS(:,:), rhoat(:) + real(dp), allocatable :: origin(:), row(:), row1(:), auxcharge(:), auxcharge1(:) + real(dp), allocatable :: g_dense(:,:),tch, Ker(:,:) + type(bml_matrix_t) :: aux_bml, dH0x_bml, dH0y_bml, dH0z_bml + type(bml_matrix_t) :: dSx_bml, dSy_bml, dSz_bml, eigenvects + type(bml_matrix_t) :: g_bml, ham0_bml, ham_bml + type(bml_matrix_t) :: over_bml, rho_bml, rhoat_bml + type(bml_matrix_t) :: rhoh_bml, zmat_bml, gch_bml + type(bml_matrix_t) :: copy_g_bml, gcov_bml, aux1_bml + type(graph_partitioning_t) :: gp + type(graph_partitioning_t) :: gpat + type(gsp2data_type) :: gsp2 + type(intpairs_type), allocatable :: intPairsH(:,:), intPairsS(:,:) + type(latte_type) :: lt + type(neighlist_type) :: nl + type(ppot_type), allocatable :: ppot(:,:) + type(sp2data_type) :: sp2 + type(system_type) :: sy + type(system_type), allocatable :: syprt(:), oldsyprt(:) + type(system_type), allocatable :: syprtk(:) + type(tbparams_type) :: tb + type(xlbo_type) :: xl + type(gpmd_type) :: gpmdt + logical :: first_part = .true. + logical :: converged = .false. + logical :: firstKernel = .true. + logical :: newPart = .false. + logical :: eig = .true. + logical :: nlistSparse = .false. + logical :: reactionDone = .false. + logical, save :: lib_init = .false. + logical, save :: lib_init2 = .false. + + type(bml_matrix_t) :: ZK1_bml, ZK2_bml, ZK3_bml + type(bml_matrix_t) :: ZK4_bml, ZK5_bml, ZK6_bml + integer :: igenz !Couter to keep track of the times zmat is computed. + logical :: tZSP, restart + type(genZSPinp) :: zsp + + integer, allocatable :: xadj_cov(:), adjncy_cov(:), CH_count_cov(:), partsSizes(:) + integer, allocatable :: part_cov(:), core_count_cov(:), Halo_count_cov(:,:) + integer, allocatable :: thisPartSize(:) + integer :: vsize(2) + integer :: nparts_cov, myRank, numRanks + integer :: getKernel_cont=0,getKernel_byBlocks_cont=0 + integer :: applyKernel_cont=0,getKernel_byParts_cont=0 + integer :: rankN_update_byParts_cont=0 + +end module gpmdcov_vars diff --git a/examples/gpmdk/src/gpmdcov_writeout.F90 b/examples/gpmdk/src/gpmdcov_writeout.F90 new file mode 100644 index 00000000..b9c897dd --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_writeout.F90 @@ -0,0 +1,245 @@ +module gpmdcov_writeout_mod + + use prg_extras_mod + use prg_system_mod + use prg_graph_mod + use bml + + public :: gpmdcov_message, gpmdcov_msI, gpmdcov_msII + public :: gpmdcov_msIII, gpmdcov_msMem, gpmdcov_msInt, gpmdcov_msVectInt, gpmdcov_msVectRel + public :: gpmdcov_writepartitionout + +contains + !> To write output file or perform some analysis + !! + subroutine gpmdcov_writepartitionout(sy,syprt,gpat,reshuffle,partsInEachRank,myRank) + use prg_partition_mod + implicit none + integer, parameter :: dp = kind(1.0d0) + type(system_type) :: sy + type(system_type), allocatable, intent(inout) :: syprt(:) + type(graph_partitioning_t), intent(inout) :: gpat + integer :: ipt,iptt,j + integer, intent(in) :: myRank + integer, allocatable, intent(in) :: partsInEachRank(:),reshuffle(:,:) + character(25) :: filename + character(20) :: auxchar + + if(allocated(sy%resindex))deallocate(sy%resindex) + allocate(sy%resindex(sy%nats)) + sy%resindex=0 + +#ifdef DO_MPI + ! !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) + do iptt=1,partsInEachRank(myRank) + ipt=reshuffle(iptt,myRank) +#else + do ipt=1,gpat%TotalParts +#endif + + write(filename,*)ipt + auxchar = adjustl(trim(filename)) + filename = "part_"//auxchar + if(.not.allocated(syprt(ipt)%net_charge)) then + allocate(syprt(ipt)%net_charge(gpat%sgraph(ipt)%llsize)) + syprt(ipt)%net_charge = 0.0_dp + endif + + call prg_write_system(syprt(ipt),filename,"pdb") + + enddo + + if(myRank == 1)then + do ipt = 1,gpat%TotalParts + do j=1,gpat%sgraph(ipt)%llsize + sy%resindex(gpat%sgraph(ipt)%core_halo_index(j)+1) = ipt + enddo + enddo + if(.not.allocated(sy%net_charge))then + allocate(sy%net_charge(sy%nats)) + sy%net_charge = 0.0_dp + endif + if(myRank == 1) call prg_write_system(sy,"system_parts","pdb") + endif + + deallocate(sy%resindex) + + end subroutine gpmdcov_writepartitionout + + subroutine gpmdcov_message(routine,message,verbose,verbTol,rank) + implicit none + integer, intent(in) :: verbose, verbTol + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + character(*), intent(in) :: routine + + if (verbose >= verbTol) then + if (present(rank)) then + if(rank == 1)then + write(*,*)"" + write(*,*)"At routine ",routine,"; ",message + write(*,*)"" + endif + else + write(*,*)"" + write(*,*)"At routine ",routine,"; ",message + write(*,*)"" + endif + endif + + end subroutine gpmdcov_message + + subroutine gpmdcov_msI(routine,message,verbose,rank) + implicit none + integer, intent(in) :: verbose + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + character(*), intent(in) :: routine + + call gpmdcov_message(routine,message,verbose,1,rank) + + end subroutine gpmdcov_msI + + subroutine gpmdcov_msII(routine,message,verbose,rank) + implicit none + integer, intent(in) :: verbose + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + character(*), intent(in) :: routine + + call gpmdcov_message(routine,message,verbose,2,rank) + + end subroutine gpmdcov_msII + + subroutine gpmdcov_msIII(routine,message,verbose,rank) + implicit none + integer, intent(in) :: verbose + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + character(*), intent(in) :: routine + + call gpmdcov_message(routine,message,verbose,3,rank) + + end subroutine gpmdcov_msIII + + subroutine gpmdcov_msMem(routine,message,verbose,rank) + implicit none + integer, intent(in) :: verbose + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + character(200) :: message1 + character(*), intent(in) :: routine + + if(verbose >= 4 .and. rank == 1)then + message1 = "At "//trim(adjustl(routine))//" "//trim(adjustl(message)) + call prg_get_mem(routine,message1) + endif + + end subroutine gpmdcov_msMem + + + subroutine gpmdcov_msRel(message,rel,verbose,verbTol,rank) + implicit none + integer, intent(in) :: verbose, verbTol + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + real(8), intent(in) :: rel + + if (verbose >= verbTol) then + if (present(rank)) then + if(rank == 1)then + write(*,*)"" + write(*,*)message," ",rel + write(*,*)"" + endif + else + write(*,*)"" + write(*,*)message," ",rel + write(*,*)"" + endif + endif + + end subroutine gpmdcov_msRel + + subroutine gpmdcov_msInt(message,inte,verbose,verbTol,rank) + implicit none + integer, intent(in) :: verbose, verbTol + integer, optional, intent(in) :: rank + character(*), intent(in) :: message + integer, intent(in) :: inte + + if (verbose >= verbTol) then + if (present(rank)) then + if(rank == 1)then + write(*,*)"" + write(*,*)message," ",inte + write(*,*)"" + endif + else + write(*,*)"" + write(*,*)message," ",inte + write(*,*)"" + endif + endif + + end subroutine gpmdcov_msInt + + subroutine gpmdcov_msVectRel(message,rel,verbose,verbTol,rank) + implicit none + integer, intent(in) :: verbose, verbTol + integer, optional, intent(in) :: rank + integer :: i + character(*), intent(in) :: message + real(8), allocatable, intent(in) :: rel(:) + + if (verbose >= verbTol) then + if (present(rank)) then + if(rank == 1)then + write(*,*)"" + write(*,*)message + do i = 1,size(rel,dim=1) + write(*,*)rel(i) + enddo + write(*,*)"" + endif + else + write(*,*)message + do i = 1,size(rel,dim=1) + write(*,*)rel(i) + enddo + write(*,*)"" + endif + endif + + end subroutine gpmdcov_msVectRel + + subroutine gpmdcov_msVectInt(message,inte,verbose,verbTol,rank) + implicit none + integer, intent(in) :: verbose, verbTol + integer, optional, intent(in) :: rank + integer :: i + character(*), intent(in) :: message + integer, allocatable, intent(in) :: inte(:) + + if (verbose >= verbTol) then + if (present(rank)) then + if(rank == 1)then + write(*,*)"" + write(*,*)message + do i = 1,size(inte,dim=1) + write(*,*)inte(i) + enddo + write(*,*)"" + endif + else + write(*,*)message + do i = 1,size(inte,dim=1) + write(*,*)inte(i) + enddo + write(*,*)"" + endif + endif + + end subroutine gpmdcov_msVectInt + +end module gpmdcov_writeout_mod diff --git a/examples/gpmdk/src/gpmdk.F90 b/examples/gpmdk/src/gpmdk.F90 new file mode 100644 index 00000000..24be5a2d --- /dev/null +++ b/examples/gpmdk/src/gpmdk.F90 @@ -0,0 +1,120 @@ +!> High-level program to perform GRAPH-BASED QMD. +!! +!! \ingroup PROGRAMS +!! \brief High-level program to perform GRAPH-BASED QMD with a DFTB Hamiltonian +!! using a full parallel Graph-based approach. +!! \note To test this program with the \verbatim run_test \endverbatim script. +!! +!! \author C. F. A. Negre +!! (cnegre@lanl.gov) +!! +!! \author S. Mniszewski +!! (smn@lanl.gov) +!! +!! \author A. M. N. Niklasson +!! (amn@lanl.gov) +!! +!! \author M. E. Wall +!! (mewall@lanl.gov) +!! +!! Verbose levels: +!! >= 0- Only print tags. +!! >= 1- Print useful scalar physical data (e.g., Total Energy) +!! >= 2- Print single file output data (e.g., 0-SCF Charges) +!! >= 3- Write out trajectory data. +!! >= 4- Write out physically meaningful 1D arrays (e.g., Charges for the species) +!! >= 5- Write out physically meaningful 2D arrays (e.g., H matrix) +!! +program gpmd + + ! Local modules + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_prepareMD_mod + use gpmdcov_mdloop_mod + use gpmdcov_init_mod + use gpmdcov_part_mod + use gpmdcov_assert_mod + use gpmdcov_mod + use gpmdcov_diagonalize_mod + use gpmdcov_kernel_mod + + ! Latte modules +#ifdef USE_LATTE + use gpmdcov_latte_mod + use setuparray + use prg_system_mod + use latteparser + use constants_mod + use bml + use nonoarray + use NEBLISTARRAY +#endif + + !integer :: ipreMD +!!!!!!!!!!!!!!!!!!!!!!!! + !> Main program driver +!!!!!!!!!!!!!!!!!!!!!!!! + + !> Initialize the program variables and parse input files. + call gpmdcov_Init(.false.) + +#ifdef USE_LATTE + call init_latte() +#endif + + call gpmdcov_assert_input(myRank) + + !We give a first guess of the Fermi level. + Ef = lt%efermi + + !The inverse of the electronic temperature. + beta = 1.0_dp/lt%kbt + + !> Initial partition of the system based on the covalency graph. + call gpmdcov_Part(1) + + if(lt%stopAt == "gpmdcov_Part") stop + + !> Initialize partitions. + call gpmdcov_InitParts() + if(lt%stopAt == "gpmdcov_InitParts") stop + + !> Comput first charges. + if(lt%method == "DiagEfFull") eig = .false. + if(.not. eig) then + call gpmdcov_Diagonalize_H0() + if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then + call gpmdcov_muFromParts() + endif + endif + call gpmdcov_FirstCharges(eig) + if(lt%stopAt == "gpmdcov_FirstCharges") stop + + + !> First SCF loop up to maxscf. + if(eig)then + call gpmdcov_DM_Min(lt%maxscf,sy%net_charge,.true.) + else + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + endif + + if(lt%stopAt == "gpmdcov_DM_Min") stop + + !> First calculation of energies and forces. + call gpmdcov_EnergAndForces(sy%net_charge) + if(lt%stopAt == "gpmdcov_Energ") stop + + !> Setup the Molecular Dynamics (MD) calculation. + call gpmdcov_PrepareMD(gpmdt%temp0) + if(lt%stopAt == "gpmdcov_PrepareMD") stop + + !> Perform the MD simulation. + call gpmdcov_MDloop() + if(lt%stopAt == "gpmdcov_MDloop") stop + + !> Finalize the program. + call gpmdcov_Finalize() + +end program gpmd diff --git a/examples/gpmdk/tests/get_energy.py b/examples/gpmdk/tests/get_energy.py new file mode 100755 index 00000000..2da423a3 --- /dev/null +++ b/examples/gpmdk/tests/get_energy.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import argparse +import re + +energy_re = re.compile("Energy Total.*=\s+([0-9-.]+)") + +parser = argparse.ArgumentParser() +parser.add_argument("OUT", + help="The output") +options = parser.parse_args() + +with open(options.OUT) as fd: + for line in fd: + result = energy_re.search(line) + if result: + print(result.group(1)) diff --git a/examples/gpmdk/tests/input_BlockPart.in b/examples/gpmdk/tests/input_BlockPart.in new file mode 100644 index 00000000..6b6ab069 --- /dev/null +++ b/examples/gpmdk/tests/input_BlockPart.in @@ -0,0 +1,55 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD_test + BMLType= Dense + Method= DiagEf + MDim= -1 + Threshold= 1.0d-5 + Verbose= 2 + MPulay= 5 + ZMat= Diag + PulayCoeff= 0.01 + MixCoeff= 0.6 + SCFTol= 1.0d-2 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.5 + MDSteps= 10 + ParamPath= "./latteTBparams" + CoordsFile= coords_300.dat + NlistEach= 1 + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -6.0 + kBT= 1.0 + Entropy= F +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + NodesPerPart= 150 + PartitionType= Block + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 5.0e-3 + ErrLimit= 1.0e-12 + PartEach= 20 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 6 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + diff --git a/examples/gpmdk/tests/input_DiagEf.in b/examples/gpmdk/tests/input_DiagEf.in new file mode 100644 index 00000000..433dc3b9 --- /dev/null +++ b/examples/gpmdk/tests/input_DiagEf.in @@ -0,0 +1,56 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD_test + BMLType= Dense + Method= DiagEf + MDim= -1 + Threshold= 1.0d-5 + Verbose= 2 + MPulay= 5 + ZMat= Diag + PulayCoeff= 0.01 + MixCoeff= 0.6 + SCFTol= 1.0d-2 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.5 + MDSteps= 10 + ParamPath= "./latteTBparams" + CoordsFile= coords_300.dat + NlistEach= 1 + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -6.0 + kBT= 1.0 + Entropy= F +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Orbital + NodesPerPart= 300 + PartitionType= METIS + NLGraphCut= 3.0 + CovGraphFact= 3.0 + PartitionCount= 4 + GraphThreshold= 5.0e-3 + ErrLimit= 1.0e-12 + PartEach= 20 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 6 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + diff --git a/examples/gpmdk/tests/input_KerByPartsBuildAlwaysMetis.in b/examples/gpmdk/tests/input_KerByPartsBuildAlwaysMetis.in new file mode 100644 index 00000000..7cca84be --- /dev/null +++ b/examples/gpmdk/tests/input_KerByPartsBuildAlwaysMetis.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= METIS + PartitionCount= 4 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= ByParts + BuildAlways= T + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerByPartsBuildOnceBlock.in b/examples/gpmdk/tests/input_KerByPartsBuildOnceBlock.in new file mode 100644 index 00000000..3a9abb83 --- /dev/null +++ b/examples/gpmdk/tests/input_KerByPartsBuildOnceBlock.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Orbital + PartitionType= Block + NodesPerPart= 12 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= ByParts + BuildAlways= F + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerByPartsBuildOnceBlockUpdate.in b/examples/gpmdk/tests/input_KerByPartsBuildOnceBlockUpdate.in new file mode 100644 index 00000000..e2b6b1cd --- /dev/null +++ b/examples/gpmdk/tests/input_KerByPartsBuildOnceBlockUpdate.in @@ -0,0 +1,85 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= Block + NodesPerPart= 12 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= ByParts + BuildAlways= F + RankNUpdate= 4 + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerByPartsBuildOnceMetisUpdate.in b/examples/gpmdk/tests/input_KerByPartsBuildOnceMetisUpdate.in new file mode 100644 index 00000000..52911720 --- /dev/null +++ b/examples/gpmdk/tests/input_KerByPartsBuildOnceMetisUpdate.in @@ -0,0 +1,85 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= METIS + PartitionCount= 4 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= ByParts + BuildAlways= F + RankNUpdate= 4 + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildAlways.in b/examples/gpmdk/tests/input_KerFullBuildAlways.in new file mode 100644 index 00000000..d591a907 --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildAlways.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= Block + NodesPerPart= 48 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= T + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildAlwaysBlock.in b/examples/gpmdk/tests/input_KerFullBuildAlwaysBlock.in new file mode 100644 index 00000000..59f3c88e --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildAlwaysBlock.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= Block + NodesPerPart= 12 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= T + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildAlwaysMetis.in b/examples/gpmdk/tests/input_KerFullBuildAlwaysMetis.in new file mode 100644 index 00000000..7c5a3472 --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildAlwaysMetis.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= METIS + PartitionCount= 4 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= T + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildOnce.in b/examples/gpmdk/tests/input_KerFullBuildOnce.in new file mode 100644 index 00000000..bf966092 --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildOnce.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= Block + NodesPerPart= 48 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= F + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildOnceBlock.in b/examples/gpmdk/tests/input_KerFullBuildOnceBlock.in new file mode 100644 index 00000000..01a90e99 --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildOnceBlock.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= Block + NodesPerPart= 12 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= F + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerFullBuildOnceMetis.in b/examples/gpmdk/tests/input_KerFullBuildOnceMetis.in new file mode 100644 index 00000000..69af304e --- /dev/null +++ b/examples/gpmdk/tests/input_KerFullBuildOnceMetis.in @@ -0,0 +1,84 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 + #SCF variables# + StopAt= "gpmdcov_DM_Min" + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-10 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 2000 + ParamPath= "./latteTBparams" + CoordsFile= "./polyaniline.pdb" + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= METIS + PartitionCount= 4 + NLGraphCut= 3.0 + CovGraphFact= 3.0 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 2 + Mdim= 200 +} + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= Full + BuildAlways= F + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_KerUpdate.in b/examples/gpmdk/tests/input_KerUpdate.in new file mode 100644 index 00000000..7137133e --- /dev/null +++ b/examples/gpmdk/tests/input_KerUpdate.in @@ -0,0 +1,76 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + BMLType= Dense + Method= DiagEfFull + MDim= -1 + Threshold= 0.0 + Verbose= 3 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + MPulay= 10 + ZMat= Diag + PulayCoeff= 0.1 + MixCoeff= 0.1 + SCFTol= 1.0d-6 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.25 + MDSteps= 20 + ParamPath= "./latteTBparams" + CoordsFile= coords_300.dat + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.5 + Entropy= T + DoKernel= T +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Atom + PartitionType= METIS + NLGraphCut= 3.0 + CovGraphFact= 3.0 + PartitionCount= 4 + GraphThreshold= 1.0e-3 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= 200 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 5 + MaxSCFIter= 0 + MaxSCFInitIter= 5 + NumThresh= 0.0 +} + +KERNEL{ + KernelType= ByParts + BuildAlways= F + RankNUpdate= 3 + KernelMixing= T + UpdateEach= 1 + Verbose= 3 +} + diff --git a/examples/gpmdk/tests/input_test.in b/examples/gpmdk/tests/input_test.in new file mode 100644 index 00000000..d2de0cee --- /dev/null +++ b/examples/gpmdk/tests/input_test.in @@ -0,0 +1,57 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD_test + BMLType= Dense + Method= DiagEf + MDim= -1 + Threshold= 1.0d-5 + Verbose= 2 + MPulay= 5 + ZMat= Diag + PulayCoeff= 0.01 + MixCoeff= 0.6 + SCFTol= 1.0d-2 + MaxSCFIter= 500 + CoulAcc= 1.0d-8 + TimeRatio= 10.0 + TimeStep= 0.5 + MDSteps= 20 + ParamPath= "../latteTBparams" + CoordsFile= coords_300.dat + NlistEach= 1 + BMLDistributionType= Distributed + MuCalcType= FromParts + EFermi= -6.0 + kBT= 1.0 + Entropy= F + #StopAt= gpmdcov_ +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + GraphElement= Orbital + NodesPerPart= 300 + PartitionType= METIS + NLGraphCut= 3.0 + CovGraphFact= 3.0 + PartitionCount= 4 + GraphThreshold= 5.0e-3 + ErrLimit= 1.0e-12 + PartEach= 20 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 6 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + diff --git a/examples/gpmdk/tests/latteTBparams/bondints.nonortho b/examples/gpmdk/tests/latteTBparams/bondints.nonortho new file mode 100755 index 00000000..2f3ba678 --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/bondints.nonortho @@ -0,0 +1,42 @@ +Noints= 39 +Element1 Element2 Kind H0 B1 B2 B3 B4 B5 R1 Rcut H0 B1 B2 B3 B4 B5 R1 Rcut +N O sss -9.360078 -1.293118 -0.379415 0.000000 0.000000 1.200000 3.500000 4.000000 0.340064 -1.703613 -0.622348 0.036738 -0.040158 1.200000 3.500000 4.000000 +N O sps 10.309052 -0.981652 -0.828497 0.000000 0.000000 1.200000 3.500000 4.000000 -0.370946 -1.040947 -0.931097 0.252441 -0.115450 1.200000 3.500000 4.000000 +O N sps 10.723048 -0.454312 -0.916563 0.000000 0.000000 1.200000 3.500000 4.000000 -0.420014 -1.107918 -0.905594 0.188424 -0.088365 1.200000 3.500000 4.000000 +N O pps 9.259131 -0.734112 -1.023762 0.000000 0.000000 1.200000 3.500000 4.000000 -0.314073 0.499050 -2.914288 2.067657 -0.738439 1.200000 3.500000 4.000000 +N O ppp -4.532623 -1.999631 -0.286275 0.000000 0.000000 1.200000 3.500000 4.000000 0.223937 -1.991867 -0.537630 -0.081270 -0.004130 1.200000 3.500000 4.000000 +C N sss -7.409712 -1.940942 -0.219762 0.000000 0.000000 1.500000 3.500000 4.000000 0.263438 -1.754525 -0.584215 -0.007801 -0.021729 1.500000 3.500000 4.000000 +C N sps 7.501761 -1.211169 -0.373905 0.000000 0.000000 1.500000 3.500000 4.000000 -0.326609 -1.197485 -0.807786 0.134891 -0.084373 1.500000 3.500000 4.000000 +N C sps 8.697591 -1.267240 -0.178484 0.000000 0.000000 1.500000 3.500000 4.000000 -0.337943 -1.335442 -0.769693 0.119373 -0.079493 1.500000 3.500000 4.000000 +C N pps 6.954600 -1.188456 -0.808043 0.000000 0.000000 1.500000 3.500000 4.000000 -0.350240 -0.467439 -1.849316 1.854403 -0.988471 1.500000 3.500000 4.000000 +C N ppp -2.921605 -2.203548 -0.409424 0.000000 0.000000 1.500000 3.500000 4.000000 0.158424 -2.114409 -0.582346 -0.051076 -0.006183 1.500000 3.500000 4.000000 +C O sss -13.986685 -1.931973 -0.432011 0.000000 0.000000 1.200000 3.500000 4.000000 0.375339 -1.547372 -0.642492 0.020614 -0.026699 1.200000 3.500000 4.000000 +C O sps 10.718738 -1.389459 -0.182128 0.000000 0.000000 1.200000 3.500000 4.000000 -0.373027 -0.776043 -1.019920 0.257539 -0.102838 1.200000 3.500000 4.000000 +O C sps 14.194791 -1.371650 -0.248285 0.000000 0.000000 1.200000 3.500000 4.000000 -0.458068 -1.035067 -0.937868 0.190562 -0.077841 1.200000 3.500000 4.000000 +C O pps 8.622023 -0.557144 -0.938551 0.000000 0.000000 1.200000 3.500000 4.000000 -0.322293 0.795473 -3.476601 2.589965 -0.897800 1.200000 3.500000 4.000000 +C O ppp -5.327397 -2.190160 -0.089303 0.000000 0.000000 1.200000 3.500000 4.000000 0.244570 -1.922717 -0.573671 -0.057280 -0.004108 1.200000 3.500000 4.000000 +N N sss -7.165811 -2.348869 -0.541905 0.000000 0.000000 1.500000 3.500000 4.000000 0.231654 -1.879002 -0.572765 -0.004579 -0.031106 1.500000 3.500000 4.000000 +N N sps 8.212268 -1.499123 -0.526440 0.000000 0.000000 1.500000 3.500000 4.000000 -0.305271 -1.385158 -0.751032 0.114531 -0.090839 1.500000 3.500000 4.000000 +N N pps 7.102331 -1.252366 -0.552533 0.000000 0.000000 1.500000 3.500000 4.000000 -0.324668 -0.547805 -1.638658 1.495168 -0.827868 1.500000 3.500000 4.000000 +N N ppp -2.828938 -2.376886 -0.560898 0.000000 0.000000 1.500000 3.500000 4.000000 0.142909 -2.162036 -0.571942 -0.071640 -0.004682 1.500000 3.500000 4.000000 +O O sss -14.387756 -2.244278 -1.645605 0.000000 0.000000 1.200000 3.500000 4.000000 0.296445 -1.911896 -0.663451 0.038054 -0.046608 1.200000 3.500000 4.000000 +O O sps 13.699127 -1.602358 -0.114474 0.000000 0.000000 1.200000 3.500000 4.000000 -0.362143 -1.285274 -0.939591 0.204641 -0.106438 1.200000 3.500000 4.000000 +O O pps 9.235469 -1.131474 -0.924535 0.000000 0.000000 1.200000 3.500000 4.000000 -0.312044 0.121814 -2.519352 1.681266 -0.644566 1.200000 3.500000 4.000000 +O O ppp -4.526526 -2.487174 -0.201464 0.000000 0.000000 1.200000 3.500000 4.000000 0.193010 -2.168462 -0.580629 -0.105104 0.004891 1.200000 3.500000 4.000000 +H O sss -12.189103 -1.800097 -0.325933 0.000000 0.000000 1.000000 3.500000 4.000000 0.404725 -1.702546 -0.707938 0.074904 -0.039922 1.000000 3.500000 4.000000 +H O sps 9.518733 -1.333235 -0.393710 0.000000 0.000000 1.000000 3.500000 4.000000 -0.447660 -0.952979 -1.163537 0.400616 -0.156965 1.000000 3.500000 4.000000 +H N sss -12.631030 -1.585597 -0.250969 0.000000 0.000000 1.000000 3.500000 4.000000 0.446693 -1.500463 -0.657448 0.065741 -0.037004 1.000000 3.500000 4.000000 +H N sps 9.837852 -1.234850 -0.324283 0.000000 0.000000 1.000000 3.500000 4.000000 -0.501530 -0.785734 -1.123232 0.394878 -0.148501 1.000000 3.500000 4.000000 +C C sss -9.197237 -1.607050 -0.535057 0.000000 0.000000 1.400000 3.500000 4.000000 0.346977 -1.519820 -0.570812 -0.013518 -0.015829 1.400000 3.500000 4.000000 +C C sps 8.562436 -0.980182 -0.646929 0.000000 0.000000 1.400000 3.500000 4.000000 -0.400467 -0.984048 -0.853949 0.157178 -0.073381 1.400000 3.500000 4.000000 +C C pps 6.614756 -0.528591 -0.951460 0.000000 0.000000 1.400000 3.500000 4.000000 -0.382417 0.102889 -2.786680 2.646356 -1.134320 1.400000 3.500000 4.000000 +C C ppp -3.678302 -1.881668 -0.255951 0.000000 0.000000 1.400000 3.500000 4.000000 0.214357 -1.948923 -0.578323 -0.034356 -0.007257 1.400000 3.500000 4.000000 +H C sss -9.235812 -1.372683 -0.408433 0.000000 0.000000 1.100000 3.500000 4.000000 0.416003 -1.459596 -0.654874 0.009140 -0.012658 1.100000 3.500000 4.000000 +H C sps 8.104851 -0.936099 -0.626219 0.000000 0.000000 1.100000 3.500000 4.000000 -0.495695 -0.901626 -1.007214 0.189808 -0.057087 1.100000 3.500000 4.000000 +H H sss -9.400000 -1.145903 -0.391777 0.000000 0.000000 0.750000 3.500000 4.000000 0.575007 -1.391261 -0.778831 0.080209 -0.017759 0.750000 3.500000 4.000000 +W W sss -2.63332044 -0.71100562 -0.27081645 0.03306840 -0.00393097 2.73 6.00 7.00 0.28895119 -0.91180491 -0.22236240 0.00062879 0.00223537 2.73 6.00 7.00 +W W sds -1.48671751 -0.42509143 -0.65004572 0.29520069 -0.06194951 2.73 6.00 7.00 0.14376383 -0.45157088 -0.59271709 0.18666281 -0.03366047 2.73 6.00 7.00 +W W dds -1.70672948 -1.10134419 -0.01972556 -0.04301193 0.00482517 2.73 6.00 7.00 0.11587117 -0.53362062 -0.52285554 0.14742450 -0.02563750 2.73 6.00 7.00 +W W ddp 1.41731714 -1.52300320 -0.03534115 -0.03114721 0.00393409 2.73 6.00 7.00 -0.11180069 -1.33376345 -0.26519523 0.05629322 -0.01154228 2.73 6.00 7.00 +W W ddd -0.32269993 -1.97500297 -0.00061059 -0.03674107 0.00594063 2.73 6.00 7.00 0.02602319 -1.93947083 -0.12231373 0.00539735 -0.00134363 2.73 6.00 7.00 +~ diff --git a/examples/gpmdk/tests/latteTBparams/bondints.nonortho.algo b/examples/gpmdk/tests/latteTBparams/bondints.nonortho.algo new file mode 100755 index 00000000..5a4554b3 --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/bondints.nonortho.algo @@ -0,0 +1,41 @@ +Noints= 39 +Element1 Element2 Kind H0 B1 B2 B3 B4 B5 R1 Rcut H0 B1 B2 B3 B4 B5 R1 Rcut +N O sss -9.360078 -1.293118 -0.379415 0.000000 0.000000 1.200000 3.500000 4.000000 0.340064 -1.703613 -0.622348 0.036738 -0.040158 1.200000 3.500000 4.000000 +N O sps 10.309052 -0.981652 -0.828497 0.000000 0.000000 1.200000 3.500000 4.000000 -0.370946 -1.040947 -0.931097 0.252441 -0.115450 1.200000 3.500000 4.000000 +O N sps 10.723048 -0.454312 -0.916563 0.000000 0.000000 1.200000 3.500000 4.000000 -0.420014 -1.107918 -0.905594 0.188424 -0.088365 1.200000 3.500000 4.000000 +N O pps 9.259131 -0.734112 -1.023762 0.000000 0.000000 1.200000 3.500000 4.000000 -0.314073 0.499050 -2.914288 2.067657 -0.738439 1.200000 3.500000 4.000000 +N O ppp -4.532623 -1.999631 -0.286275 0.000000 0.000000 1.200000 3.500000 4.000000 0.223937 -1.991867 -0.537630 -0.081270 -0.004130 1.200000 3.500000 4.000000 +C N sss -7.409712 -1.940942 -0.219762 0.000000 0.000000 1.500000 3.500000 4.000000 0.263438 -1.754525 -0.584215 -0.007801 -0.021729 1.500000 3.500000 4.000000 +C N sps 7.501761 -1.211169 -0.373905 0.000000 0.000000 1.500000 3.500000 4.000000 -0.326609 -1.197485 -0.807786 0.134891 -0.084373 1.500000 3.500000 4.000000 +N C sps 8.697591 -1.267240 -0.178484 0.000000 0.000000 1.500000 3.500000 4.000000 -0.337943 -1.335442 -0.769693 0.119373 -0.079493 1.500000 3.500000 4.000000 +C N pps 6.954600 -1.188456 -0.808043 0.000000 0.000000 1.500000 3.500000 4.000000 -0.350240 -0.467439 -1.849316 1.854403 -0.988471 1.500000 3.500000 4.000000 +C N ppp -2.921605 -2.203548 -0.409424 0.000000 0.000000 1.500000 3.500000 4.000000 0.158424 -2.114409 -0.582346 -0.051076 -0.006183 1.500000 3.500000 4.000000 +C O sss -13.986685 -1.931973 -0.432011 0.000000 0.000000 1.200000 3.500000 4.000000 0.375339 -1.547372 -0.642492 0.020614 -0.026699 1.200000 3.500000 4.000000 +C O sps 10.718738 -1.389459 -0.182128 0.000000 0.000000 1.200000 3.500000 4.000000 -0.373027 -0.776043 -1.019920 0.257539 -0.102838 1.200000 3.500000 4.000000 +O C sps 14.194791 -1.371650 -0.248285 0.000000 0.000000 1.200000 3.500000 4.000000 -0.458068 -1.035067 -0.937868 0.190562 -0.077841 1.200000 3.500000 4.000000 +C O pps 8.622023 -0.557144 -0.938551 0.000000 0.000000 1.200000 3.500000 4.000000 -0.322293 0.795473 -3.476601 2.589965 -0.897800 1.200000 3.500000 4.000000 +C O ppp -5.327397 -2.190160 -0.089303 0.000000 0.000000 1.200000 3.500000 4.000000 0.244570 -1.922717 -0.573671 -0.057280 -0.004108 1.200000 3.500000 4.000000 +N N sss -7.165811 -2.348869 -0.541905 0.000000 0.000000 1.500000 3.500000 4.000000 0.231654 -1.879002 -0.572765 -0.004579 -0.031106 1.500000 3.500000 4.000000 +N N sps 8.212268 -1.499123 -0.526440 0.000000 0.000000 1.500000 3.500000 4.000000 -0.305271 -1.385158 -0.751032 0.114531 -0.090839 1.500000 3.500000 4.000000 +N N pps 7.102331 -1.252366 -0.552533 0.000000 0.000000 1.500000 3.500000 4.000000 -0.324668 -0.547805 -1.638658 1.495168 -0.827868 1.500000 3.500000 4.000000 +N N ppp -2.828938 -2.376886 -0.560898 0.000000 0.000000 1.500000 3.500000 4.000000 0.142909 -2.162036 -0.571942 -0.071640 -0.004682 1.500000 3.500000 4.000000 +O O sss -14.387756 -2.244278 -1.645605 0.000000 0.000000 1.200000 3.500000 4.000000 0.296445 -1.911896 -0.663451 0.038054 -0.046608 1.200000 3.500000 4.000000 +O O sps 13.699127 -1.602358 -0.114474 0.000000 0.000000 1.200000 3.500000 4.000000 -0.362143 -1.285274 -0.939591 0.204641 -0.106438 1.200000 3.500000 4.000000 +O O pps 9.235469 -1.131474 -0.924535 0.000000 0.000000 1.200000 3.500000 4.000000 -0.312044 0.121814 -2.519352 1.681266 -0.644566 1.200000 3.500000 4.000000 +O O ppp -4.526526 -2.487174 -0.201464 0.000000 0.000000 1.200000 3.500000 4.000000 0.193010 -2.168462 -0.580629 -0.105104 0.004891 1.200000 3.500000 4.000000 +H O sss -12.189103 -1.800097 -0.325933 0.000000 0.000000 1.000000 3.500000 4.000000 0.404725 -1.702546 -0.707938 0.074904 -0.039922 1.000000 3.500000 4.000000 +H O sps 9.518733 -1.333235 -0.393710 0.000000 0.000000 1.000000 3.500000 4.000000 -0.447660 -0.952979 -1.163537 0.400616 -0.156965 1.000000 3.500000 4.000000 +H N sss -12.631030 -1.585597 -0.250969 0.000000 0.000000 1.000000 3.500000 4.000000 0.446693 -1.500463 -0.657448 0.065741 -0.037004 1.000000 3.500000 4.000000 +H N sps 9.837852 -1.234850 -0.324283 0.000000 0.000000 1.000000 3.500000 4.000000 -0.501530 -0.785734 -1.123232 0.394878 -0.148501 1.000000 3.500000 4.000000 +C C sss -9.197237 -1.607050 -0.535057 0.000000 0.000000 1.400000 3.500000 4.000000 0.346977 -1.519820 -0.570812 -0.013518 -0.015829 1.400000 3.500000 4.000000 +C C sps 8.562436 -0.980182 -0.646929 0.000000 0.000000 1.400000 3.500000 4.000000 -0.400467 -0.984048 -0.853949 0.157178 -0.073381 1.400000 3.500000 4.000000 +C C pps 6.614756 -0.528591 -0.951460 0.000000 0.000000 1.400000 3.500000 4.000000 -0.382417 0.102889 -2.786680 2.646356 -1.134320 1.400000 3.500000 4.000000 +C C ppp -3.678302 -1.881668 -0.255951 0.000000 0.000000 1.400000 3.500000 4.000000 0.214357 -1.948923 -0.578323 -0.034356 -0.007257 1.400000 3.500000 4.000000 +H C sss -9.235812 -1.372683 -0.408433 0.000000 0.000000 1.100000 3.500000 4.000000 0.416003 -1.459596 -0.654874 0.009140 -0.012658 1.100000 3.500000 4.000000 +H C sps 8.104851 -0.936099 -0.626219 0.000000 0.000000 1.100000 3.500000 4.000000 -0.495695 -0.901626 -1.007214 0.189808 -0.057087 1.100000 3.500000 4.000000 +H H sss -9.400000 -1.145903 -0.391777 0.000000 0.000000 0.750000 3.500000 4.000000 0.575007 -1.391261 -0.778831 0.080209 -0.017759 0.750000 3.500000 4.000000 +W W sss -2.63332044 -0.71100562 -0.27081645 0.03306840 -0.00393097 2.73 6.00 7.00 0.28895119 -0.91180491 -0.22236240 0.00062879 0.00223537 2.73 6.00 7.00 +W W sds -1.48671751 -0.42509143 -0.65004572 0.29520069 -0.06194951 2.73 6.00 7.00 0.14376383 -0.45157088 -0.59271709 0.18666281 -0.03366047 2.73 6.00 7.00 +W W dds -1.70672948 -1.10134419 -0.01972556 -0.04301193 0.00482517 2.73 6.00 7.00 0.11587117 -0.53362062 -0.52285554 0.14742450 -0.02563750 2.73 6.00 7.00 +W W ddp 1.41731714 -1.52300320 -0.03534115 -0.03114721 0.00393409 2.73 6.00 7.00 -0.11180069 -1.33376345 -0.26519523 0.05629322 -0.01154228 2.73 6.00 7.00 +W W ddd -0.32269993 -1.97500297 -0.00061059 -0.03674107 0.00594063 2.73 6.00 7.00 0.02602319 -1.93947083 -0.12231373 0.00539735 -0.00134363 2.73 6.00 7.00 diff --git a/examples/gpmdk/tests/latteTBparams/bondints.ortho b/examples/gpmdk/tests/latteTBparams/bondints.ortho new file mode 100755 index 00000000..098aad6a --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/bondints.ortho @@ -0,0 +1,53 @@ +Noints= 51 +Element1 Element2 Kind H0 B1 B2 B3 B4 B5 R1 Rcut + C N sss -5.3 -1.4 -0.2 0.0 0.0 1.25 1.7 2.7 + C N sps 3.5 -1.8 -0.2 0.0 0.0 1.25 1.7 2.7 + N C sps 4.6 -1.8 -0.2 0.0 0.0 1.25 1.7 2.7 + C N pps 6.2 -2.0 -0.2 0.0 0.0 1.25 1.7 2.7 + C N ppp -3.0 -2.0 -0.2 0.0 0.0 1.25 1.7 2.7 + N N sss -7.5 -1.60 -0.2 0.0 0.0 1.1 1.5 2.5 + N N sps 7.5 -1.5 -0.2 0.0 0.0 1.1 1.5 2.5 + N N pps 4.6 -1.25 -0.2 0.0 0.0 1.1 1.5 2.5 + N N ppp -3.5 -2.02 0.0 0.0 0.0 1.1 1.5 2.5 + H N sss -7.5 -1.5 -0.2 0.0 0.0 1.0 1.2 2.2 + H N sps 6.3 -1.5 -0.2 0.0 0.0 1.0 1.2 2.2 + H H sss -5.1 -1.30 -0.25 0.0 0.0 0.74 1.2 2.2 + H C sss -6.0 -1.3 -0.5 0.0 0.0 1.1 1.25 2.3 + H C sps 3.75 -1.3 -0.5 0.0 0.0 1.1 1.25 2.3 + O O sss -5.0 -2.1 -0.3 0.0 0.0 1.2 1.5 2.3 + O O sps 5.9 -2.1 -0.3 0.0 0.0 1.2 1.5 2.3 + O O pps 8.3 -2.1 -0.3 0.0 0.0 1.2 1.5 2.3 + O O ppp -4.0 -2.1 -0.3 0.0 0.0 1.2 1.5 2.3 + C C sss -3.8 -1.4 -0.2 0.0 0.0 1.55 1.7 2.7 + C C sps 4.5 -1.8 -0.2 0.0 0.0 1.55 1.7 2.7 + C C pps 5.6 -2.0 -0.2 0.0 0.0 1.55 1.7 2.7 + C C ppp -1.75 -2.0 -0.2 0.0 0.0 1.55 1.7 2.7 + N O sss -6.4 -1.85 -0.25 0.0 0.0 1.2 1.5 2.5 + N O sps 6.15 -1.5 -0.3 0.0 0.0 1.2 1.5 2.5 + O N sps 7.25 -1.5 -0.3 0.0 0.0 1.2 1.5 2.5 + N O pps 6.5 -1.5 -0.3 0.0 0.0 1.2 1.5 2.5 + N O ppp -2.75 -2.06 0.0 0.0 0.0 1.2 1.7 2.5 + C O sss -6.8 -1.7 -0.3 0.0 0.0 1.15 1.7 2.5 + C O sps 4.7 -1.7 -0.3 0.0 0.0 1.15 1.7 2.5 + O C sps 7.7 -1.7 -0.3 0.0 0.0 1.15 1.7 2.5 + C O pps 3.8 -1.7 -0.3 0.0 0.0 1.15 1.7 2.5 + C O ppp -3.2 -1.7 -0.3 0.0 0.0 1.15 1.7 2.5 + O U sss -4.1203 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + O U sds -3.7902 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + O U sfs 2.51027 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + U O sps 3.61703 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + O U pds -5.73676 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + O U pdp 4.00022 -2.2 -0.5 0.0 0.0 1.78 1.9 2.7 + O U pfs 6.6 -2.4 -0.5 0.0 0.0 1.78 1.9 2.7 + O U pfp -4.51041 -2.4 -0.5 0.0 0.0 1.78 1.9 2.7 + H U sss -4.5 -2.3 -0.5 -0.2 0.0 2.03 2.2 2.75 + H U sps 4.8 -2.3 -0.5 -0.2 0.0 2.03 2.2 2.75 + H U sds -1.25 -2.3 -0.5 -0.2 0.0 2.03 2.2 2.75 + H U sfs 0.65 -2.3 -0.5 -0.2 0.0 2.03 2.2 2.75 + H O sss -5.4 -1.8 -0.3 0.0 0.0 1.0 1.2 2.2 + H O sps 5.0 -1.8 -0.3 0.0 0.0 1.0 1.2 2.2 + Ti Ti sss -0.375 -1.0 -0.1 0.0 0.0 2.89 3.2 4.4 + Ti Ti sds -0.375 -1.0 -0.1 0.0 0.0 2.89 3.2 4.4 + Ti Ti dds -0.82 -1.25 -0.20 0.0 0.0 2.89 3.2 4.4 + Ti Ti ddp 0.57 -1.25 -0.25 0.0 0.0 2.89 3.2 4.4 + Ti Ti ddd -0.15 -1.05 -0.2 0.0 0.0 2.89 3.2 4.4 diff --git a/examples/gpmdk/tests/latteTBparams/electrons.dat.algo b/examples/gpmdk/tests/latteTBparams/electrons.dat.algo new file mode 100755 index 00000000..9946312c --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/electrons.dat.algo @@ -0,0 +1,7 @@ +Noelem= 5 +Element basis Numel Es Ep Ed Ef Mass HubbardU Wss Wpp Wdd Wff +N sp 5.000000 -18.556500 -7.062500 0.000000 0.000000 14.006700 17.372900 0.000000 -0.693400 0.000000 0.000000 +O sp 6.000000 -23.937700 -9.003500 0.000000 0.000000 15.999400 11.876141 0.000000 -0.7576500 0.000000 0.000000 +H s 1.000000 -6.483500 0.000000 0.000000 0.000000 1.007900 12.054683 -2.23400 0.000000 0.000000 0.000000 +C sp 4.000000 -13.719900 -5.254100 0.000000 0.000000 12.010000 14.240811 0.000000 -0.6181000 0.000000 0.000000 +W sd 6.0 -4.05 0.0 -2.12 0.0 183.84 7.048 0.0 0.0 0.0 0.0 diff --git a/examples/gpmdk/tests/latteTBparams/ppots.nonortho b/examples/gpmdk/tests/latteTBparams/ppots.nonortho new file mode 100755 index 00000000..63d1feac --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/ppots.nonortho @@ -0,0 +1,13 @@ +Nopps= 10 +Ele1 Ele2 A0 A1 A2 A3 A4 A5 A6 C R1 Rcut +C C 3.927770 24.439989 -51.156433 39.032536 -11.321277 0.000000 0.000000 0.000000 1.600000 1.700000 +N O 14.005908 19.769009 -46.607006 38.399015 -12.656658 0.000000 0.000000 0.000000 1.600000 1.700000 +C N 98.283078 10.289077 -27.709052 22.099235 -6.796462 0.000000 0.000000 0.000000 1.600000 1.700000 +N N 40.335850 14.958977 -36.644093 29.219613 -8.918783 0.000000 0.000000 0.000000 1.600000 1.700000 +C O 0.916287 30.115416 -59.612502 45.114207 -13.200384 0.000000 0.000000 0.000000 1.500000 1.600000 +O O 11.833452 19.281518 -45.763767 37.924165 -12.006535 0.000000 0.000000 0.000000 1.500000 1.600000 +N H 0.664002 28.086622 -63.415978 53.301425 -17.343446 0.000000 0.000000 0.000000 1.300000 1.400000 +O H 0.484351 33.176296 -81.154354 74.931992 -26.796460 0.000000 0.000000 0.000000 1.200000 1.300000 +C H 1.094168 28.606497 -71.558353 65.967464 -23.372892 0.000000 0.000000 0.000000 1.200000 1.300000 +H H 8.194700 16.371100 -75.246500 106.703000 -59.105700 0.000000 0.000000 0.000000 0.800000 0.900000 +~ diff --git a/examples/gpmdk/tests/latteTBparams/ppots.nonortho.algo b/examples/gpmdk/tests/latteTBparams/ppots.nonortho.algo new file mode 100755 index 00000000..6d0f8061 --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/ppots.nonortho.algo @@ -0,0 +1,12 @@ +Nopps= 10 +Ele1 Ele2 A0 A1 A2 A3 A4 A5 A6 C R1 Rcut +C C 3.927770 24.439989 -51.156433 39.032536 -11.321277 0.000000 0.000000 0.000000 1.600000 1.700000 +N O 14.005908 19.769009 -46.607006 38.399015 -12.656658 0.000000 0.000000 0.000000 1.600000 1.700000 +C N 98.283078 10.289077 -27.709052 22.099235 -6.796462 0.000000 0.000000 0.000000 1.600000 1.700000 +N N 40.335850 14.958977 -36.644093 29.219613 -8.918783 0.000000 0.000000 0.000000 1.600000 1.700000 +C O 0.916287 30.115416 -59.612502 45.114207 -13.200384 0.000000 0.000000 0.000000 1.500000 1.600000 +O O 11.833452 19.281518 -45.763767 37.924165 -12.006535 0.000000 0.000000 0.000000 1.500000 1.600000 +N H 0.664002 28.086622 -63.415978 53.301425 -17.343446 0.000000 0.000000 0.000000 1.300000 1.400000 +O H 0.484351 33.176296 -81.154354 74.931992 -26.796460 0.000000 0.000000 0.000000 1.200000 1.300000 +C H 1.094168 28.606497 -71.558353 65.967464 -23.372892 0.000000 0.000000 0.000000 1.200000 1.300000 +H H 8.194700 16.371100 -75.246500 106.703000 -59.105700 0.000000 0.000000 0.000000 0.800000 0.900000 diff --git a/examples/gpmdk/tests/latteTBparams/ppots.ortho b/examples/gpmdk/tests/latteTBparams/ppots.ortho new file mode 100755 index 00000000..6ce50391 --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/ppots.ortho @@ -0,0 +1,15 @@ +Nopps= 13 +Ele1 Ele2 A0 A1 A2 A3 A4 A5 A6 C R1 Rcut +C N -20.6799 12.9177 -19.6308 10.0147 -1.99996 11444.6 -4.50837 20.91 6.0 8.0 +N N -0.00022258 39.5237 -44.8287 20.7267 -3.63323 3470.99 -3.9 17.29 6.0 8.0 +H N 27.5208 22.4977 -53.2646 36.489 -8.14765 310.995 -3.25 4.97 6.0 8.0 +H H 153760.0 -19.5418 10.4692 -2.36546 -4.56753e-8 31.8014 -3.3 1.43 6.0 8.0 +H O 35415.8 -12.1641 4.33584 -0.665623 -1.57864e-8 66.9731 -3.2 4.64 6.0 8.0 +O O -1.50948 24.9176 -43.551 29.4145 -7.67058 5860.13 -4.2 15.06 6.0 8.0 +C C -285.763 8.57988 -15.515 8.4654 -1.82402 20501.9 -4.3 25.28 6.0 8.0 +N O -951.66 2.7816 -9.50681 5.89029 -1.5033 11009.4 -4.4 16.14 6.0 8.0 +C H 4.50718e6 -27.2903 21.0623 -8.35436 -0.00014673 312.288 -3.5 6.01 6.0 8.0 +C O 26.8926 17.7353 -35.7842 23.6613 -5.86093 548.118 -3.3 19.52 6.0 8.0 +U O 9.01849e-5 51.1656 -57.7771 26.5256 -4.46059 -8.63768 -0.8 50.0 5.0 6.0 +U H 4948.73 6.41405 -18.2323 10.9876 -2.17677 5.8963 -1.1 0.0 4.0 6.0 +Ti Ti -0.000257043 17.2475 -10.5778 2.63878 -0.238222 1250.77 -2.35513 0.0 6.0 8.0 \ No newline at end of file diff --git a/examples/gpmdk/tests/ref.BlockPart b/examples/gpmdk/tests/ref.BlockPart new file mode 100644 index 00000000..131380eb --- /dev/null +++ b/examples/gpmdk/tests/ref.BlockPart @@ -0,0 +1,10 @@ +-1411.0490851791503 +-1411.1895075798777 +-1411.2094225665480 +-1411.2335956442234 +-1411.2542969289500 +-1411.2669052025312 +-1411.2729283760630 +-1411.2675447279273 +-1411.2699688505029 +-1411.2737450666507 diff --git a/examples/gpmdk/tests/ref.DiagEf b/examples/gpmdk/tests/ref.DiagEf new file mode 100644 index 00000000..131380eb --- /dev/null +++ b/examples/gpmdk/tests/ref.DiagEf @@ -0,0 +1,10 @@ +-1411.0490851791503 +-1411.1895075798777 +-1411.2094225665480 +-1411.2335956442234 +-1411.2542969289500 +-1411.2669052025312 +-1411.2729283760630 +-1411.2675447279273 +-1411.2699688505029 +-1411.2737450666507 diff --git a/examples/gpmdk/tests/ref.KerUpdate b/examples/gpmdk/tests/ref.KerUpdate new file mode 100644 index 00000000..d22666f0 --- /dev/null +++ b/examples/gpmdk/tests/ref.KerUpdate @@ -0,0 +1,101 @@ +-1371.7258405867140 +-1371.7253134129903 +-1371.7261717318202 +-1371.7275198254608 +-1371.7292398457998 +-1371.7313197915726 +-1371.7329353059108 +-1371.7341277155258 +-1371.7347540551209 +-1371.7348134149959 +-1371.7344152720541 +-1371.7336777542523 +-1371.7326655692327 +-1371.7313724946732 +-1371.7297824725022 +-1371.7279655016523 +-1371.7261429622149 +-1371.7246712301069 +-1371.7239361640861 +-1371.7242067506304 +-1371.7255216559779 +-1371.7276664507845 +-1371.7302484255763 +-1371.7328279703561 +-1371.7350364715428 +-1371.7366342028533 +-1371.7375004339860 +-1371.7375904283697 +-1371.7369052881700 +-1371.7355020959997 +-1371.7335296296355 +-1371.7312490513732 +-1371.7290019913378 +-1371.7271232970929 +-1371.7258366183116 +-1371.7251823856091 +-1371.7250401801182 +-1371.7252194860553 +-1371.7255865840573 +-1371.7261402707566 +-1371.7269972069985 +-1371.7282904810893 +-1371.7300465912097 +-1371.7321033888975 +-1371.7341298896974 +-1371.7357267237210 +-1371.7365657268322 +-1371.7364887730334 +-1371.7355325222416 +-1371.7338831452028 +-1371.7318025464560 +-1371.7295739612709 +-1371.7274839915417 +-1371.7258201288655 +-1371.7248569429553 +-1371.7248037328582 +-1371.7257279589187 +-1371.7274958644120 +-1371.7297753937348 +-1371.7321176270282 +-1371.7340934003098 +-1371.7354216036085 +-1371.7360312167316 +-1371.7360279846116 +-1371.7355932623852 +-1371.7348708337852 +-1371.7339102200510 +-1371.7326939919412 +-1371.7312186220493 +-1371.7295781391451 +-1371.7279863356362 +-1371.7267224516886 +-1371.7260304672168 +-1371.7260289899043 +-1371.7266819243791 +-1371.7278387350050 +-1371.7293112748428 +-1371.7309346476675 +-1371.7325801859488 +-1371.7341221892621 +-1371.7353988323553 +-1371.7362091280411 +-1371.7363629211600 +-1371.7357623335529 +-1371.7344616377115 +-1371.7326725469131 +-1371.7307000959938 +-1371.7288467356693 +-1371.7273295395462 +-1371.7262555496391 +-1371.7256545345795 +-1371.7255377506356 +-1371.7259356298337 +-1371.7268862303183 +-1371.7283786154999 +-1371.7302944926196 +-1371.7323904507596 +-1371.7343450304184 +-1371.7358522937034 +-1371.7367147534512 +-1371.7368882589715 diff --git a/examples/gpmdk/tests/ref.gpmdcov_DM_Min b/examples/gpmdk/tests/ref.gpmdcov_DM_Min new file mode 100644 index 00000000..fcf80638 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_DM_Min @@ -0,0 +1,1705 @@ + + Building covalency graph ... + + In graph_part .. + prg_metisPartition_test start ... + + total procs = 1 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 4 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 4 localPartExtent = 4 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + MPI rank 1 done with graph_part .. + 1 1 1 265 + 1 2 2 271 + 1 3 3 277 + 1 4 4 262 + Cost per rank = 77750796.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.0000000000000000 + + At routine gpmdcov_Part; Getting CH subsystems ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =9.89084887E-02 + + + Total charge of the system = -1.28518280E+01 + + + Full System charges: + 1 H -0.38287587677689805 + 2 O 0.63744382810575040 + 3 H -0.38541139381519052 + 4 H -0.38651471542141813 + 5 O 0.66237290215892841 + 6 H -0.38504033812318228 + 7 H -0.38540738716704459 + 8 O 0.63787141862758556 + 9 H -0.38597359710200418 + 10 H -0.38757747967414991 + 11 O 0.62807003872116951 + 12 H -0.38682491795099527 + 13 H -0.38410222180602738 + 14 O 0.62461763109826052 + 15 H -0.38698393957444754 + 16 H -0.38440149425428360 + 17 O 0.61164053895860881 + 18 H -0.38569277579158856 + 19 H -0.38434232313365257 + 20 O 0.64463111078439894 + 21 H -0.38462567965123717 + 22 H -0.38427141445758162 + 23 O 0.63690730745572566 + 24 H -0.38291322041581344 + 25 H -0.38381522318335359 + 26 O 0.62992631464225912 + 27 H -0.38350999167534283 + 28 H -0.38880110696455972 + 29 O 0.69204678733768432 + 30 H -0.38488397141888431 + 31 H -0.38614529229822347 + 32 O 0.67075350252162025 + 33 H -0.38404784601704689 + 34 H -0.38452544640896313 + 35 O 0.63230105859271823 + 36 H -0.38575915800819027 + 37 H -0.38602977132028071 + 38 O 0.63606304782821699 + 39 H -0.38378765834477224 + 40 H -0.38403006238803217 + 41 O 0.63075326699853118 + 42 H -0.38514867798428343 + 43 H -0.38398696781303099 + 44 O 0.61860940988109103 + 45 H -0.38379573523016774 + 46 H -0.38482235273610799 + 47 O 0.63819219704432051 + 48 H -0.38509705907999980 + 49 H -0.38317111858896657 + 50 O 0.64160880563308798 + 51 H -0.38267976529147196 + 52 H -0.38373788242832441 + 53 O 0.65612766765774122 + 54 H -0.38563068856926264 + 55 H -0.38797000375734991 + 56 O 0.66220650748357190 + 57 H -0.38652245118314088 + 58 H -0.38483444937967293 + 59 O 0.63561678024539070 + 60 H -0.38513110875621692 + 61 H -0.38313463961007366 + 62 O 0.63369824633151861 + 63 H -0.38450553325596992 + 64 H -0.38432277573572049 + 65 O 0.61941334656292568 + 66 H -0.38377026940288972 + 67 H -0.39115959546148182 + 68 O 0.65138540960355229 + 69 H -0.39150677342225915 + 70 H -0.38789877451764865 + 71 O 0.63429619996013287 + 72 H -0.38904257569339584 + 73 H -0.38425170596187530 + 74 O 0.63725221410264066 + 75 H -0.38355865624878416 + 76 H -0.38685532411767809 + 77 O 0.65232539520473498 + 78 H -0.38798279835701854 + 79 H -0.38411042895466341 + 80 O 0.65773905619248385 + 81 H -0.38505165583432333 + 82 H -0.38198315361624835 + 83 O 0.60533181385956247 + 84 H -0.38273469215827438 + 85 H -0.38489764843445495 + 86 O 0.63465542092936555 + 87 H -0.38749414267057036 + 88 H -0.38522955554360816 + 89 O 0.63690301353478329 + 90 H -0.38346289815765888 + 91 H -0.38262803902409004 + 92 O 0.64189293217961030 + 93 H -0.38304949129675059 + 94 H -0.38337247651731654 + 95 O 0.66069899866248338 + 96 H -0.38716379942533952 + 97 H -0.38788068477271853 + 98 O 0.62264614994751000 + 99 H -0.38687439032101423 + 100 H -0.38611348446694038 + 101 O 0.62438536664749922 + 102 H -0.38461511378639046 + 103 H -0.38526737494943364 + 104 O 0.65027032582021960 + 105 H -0.38644010140223906 + 106 H -0.38409110158860615 + 107 O 0.66003540816961159 + 108 H -0.38413514424952078 + 109 H -0.38547069494616037 + 110 O 0.62418886446238098 + 111 H -0.38295473027726701 + 112 H -0.38119891581465293 + 113 O 0.61937073804837794 + 114 H -0.38105141154062239 + 115 H -0.38762340720661848 + 116 O 0.64114078330534863 + 117 H -0.38590144632591938 + 118 H -0.38425045258382484 + 119 O 0.63118692474923233 + 120 H -0.38301206157437906 + 121 H -0.38494357736923623 + 122 O 0.64077483925598244 + 123 H -0.38491437791989880 + 124 H -0.38555699098850948 + 125 O 0.66441421855959248 + 126 H -0.38399860161497479 + 127 H -0.38695926376822043 + 128 O 0.64622819589657521 + 129 H -0.39086264883170607 + 130 H -0.38781529207117504 + 131 O 0.66359286855500699 + 132 H -0.38635402655560647 + 133 H -0.38422236589019254 + 134 O 0.60880925933512930 + 135 H -0.38522178821507225 + 136 H -0.38318725961900657 + 137 O 0.63999436963939438 + 138 H -0.38503364424014719 + 139 H -0.38444827319404984 + 140 O 0.63162705423999821 + 141 H -0.38685743390000493 + 142 H -0.38438048119938972 + 143 O 0.65037371782720754 + 144 H -0.38556147033059696 + 145 H -0.38659418389048872 + 146 O 0.65153713214770104 + 147 H -0.38348876245634655 + 148 H -0.38572408579796158 + 149 O 0.65773120839195442 + 150 H -0.38539183932618937 + 151 H -0.38397021974045698 + 152 O 0.65009790851119931 + 153 H -0.38441385106118808 + 154 H -0.38389264712556015 + 155 O 0.63320965652431482 + 156 H -0.38432831606532147 + 157 H -0.38458175026261310 + 158 O 0.63542657721683415 + 159 H -0.38511740457510302 + 160 H -0.38757841214223698 + 161 O 0.66869062025316683 + 162 H -0.38617005218926670 + 163 H -0.38695138354532588 + 164 O 0.63727303882707620 + 165 H -0.38574950053513923 + 166 H -0.38575810154876411 + 167 O 0.61333574738949359 + 168 H -0.38255353360042155 + 169 H -0.38671303493857589 + 170 O 0.61045249942596946 + 171 H -0.38365226372453798 + 172 H -0.38547156658637960 + 173 O 0.63302050421687994 + 174 H -0.38364197366170583 + 175 H -0.38236443078989413 + 176 O 0.64653045030051270 + 177 H -0.38254487328351627 + 178 H -0.39676353193869096 + 179 O 0.63111758741108570 + 180 H -0.39116243200286149 + 181 H -0.38166561972706725 + 182 O 0.64780287840810935 + 183 H -0.38260631132431167 + 184 H -0.38476220626925972 + 185 O 0.62211481914551481 + 186 H -0.38381531485769760 + 187 H -0.38799420545609231 + 188 O 0.66171710360223379 + 189 H -0.38640461442453788 + 190 H -0.38215089085834897 + 191 O 0.65257516641836677 + 192 H -0.38317921215679529 + 193 H -0.38798578773062242 + 194 O 0.68209216708275378 + 195 H -0.39714375933507828 + 196 H -0.38547191962063021 + 197 O 0.64336181483855359 + 198 H -0.38651757668281683 + 199 H -0.38276486846342583 + 200 O 0.64958033572627194 + 201 H -0.38085267905139586 + 202 H -0.38338731812501481 + 203 O 0.61119591206047108 + 204 H -0.38306285943501095 + 205 H -0.38299499502653556 + 206 O 0.61515044931978125 + 207 H -0.38356168980813699 + 208 H -0.38388172556888955 + 209 O 0.64211704076982201 + 210 H -0.38408151631059828 + 211 H -0.38500225471000726 + 212 O 0.64873254900008526 + 213 H -0.38290771103533794 + 214 H -0.38757256078164437 + 215 O 0.65786661767194854 + 216 H -0.38758208258894811 + 217 H -0.38833119113424874 + 218 O 0.60323454233107032 + 219 H -0.38236661602340249 + 220 H -0.38433613423468960 + 221 O 0.62409552606179552 + 222 H -0.38737576456748191 + 223 H -0.38293757247369964 + 224 O 0.60869696050420607 + 225 H -0.38326636945256198 + 226 H -0.38564133238539988 + 227 O 0.66763155712650857 + 228 H -0.38687397090242870 + 229 H -0.38646215478652435 + 230 O 0.65757958201066646 + 231 H -0.38385699218984004 + 232 H -0.38190000073041175 + 233 O 0.63852217728445559 + 234 H -0.38513060512857245 + 235 H -0.38428347698072030 + 236 O 0.61350492733636663 + 237 H -0.38468182639188897 + 238 H -0.38539078054099118 + 239 O 0.65334237266432993 + 240 H -0.38581660018478026 + 241 H -0.38529378655888213 + 242 O 0.65313419778199489 + 243 H -0.38520359113499614 + 244 H -0.38398297014967386 + 245 O 0.62491894103860712 + 246 H -0.38491811026513045 + 247 H -0.38439931796994042 + 248 O 0.66442688004916750 + 249 H -0.38430862324754278 + 250 H -0.38348371358233035 + 251 O 0.64037511754996856 + 252 H -0.38474205159406361 + 253 H -0.38266857559150347 + 254 O 0.63505651351883152 + 255 H -0.38467157908611060 + 256 H -0.38676877538739041 + 257 O 0.64637597010395798 + 258 H -0.38609162828711374 + 259 H -0.38388924763936738 + 260 O 0.65844589631668704 + 261 H -0.38281724522328220 + 262 H -0.38481306309278263 + 263 O 0.65662742792211315 + 264 H -0.38469109720402017 + 265 H -0.38448488320095653 + 266 O 0.65909431812519870 + 267 H -0.38563528522701196 + 268 H -0.38493270943074420 + 269 O 0.63772861798929981 + 270 H -0.38399947001702261 + 271 H -0.38360906525456329 + 272 O 0.64715559191300542 + 273 H -0.38481490025647014 + 274 H -0.38383142679871352 + 275 O 0.65983854464200409 + 276 H -0.38603396323834538 + 277 H -0.38308994208739355 + 278 O 0.63603608218827556 + 279 H -0.38692798845397547 + 280 H -0.38864604877136599 + 281 O 0.62828867865037008 + 282 H -0.39580455241394807 + 283 H -0.38853989357943242 + 284 O 0.66894011974030043 + 285 H -0.38835593846051564 + 286 H -0.38412741456803612 + 287 O 0.65787668243718400 + 288 H -0.38435435026896658 + 289 H -0.39067660690435058 + 290 O 0.65260435683509499 + 291 H -0.38921350126075760 + 292 H -0.38337851791964173 + 293 O 0.64876419705635957 + 294 H -0.38255343486336202 + 295 H -0.38456644513480631 + 296 O 0.63383606799740377 + 297 H -0.38380323087890689 + 298 H -0.38730361473968378 + 299 O 0.70697947795857807 + 300 H -0.39086661652135513 + SCF iter 1 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 1 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 4 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 9.8908488666986233E-002 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 9.8908488666986233E-002 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 9.8908488666986233E-002 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 9.8908488666986233E-002 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + SCF error = 1.1508290370954896E-002 + + At routine gpmdcov_dm_min; Total charge =-2.81983066E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.78384860E+00 + + SCF iter 2 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 2 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.7838485996303997 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.7838485996303997 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.7838485996303997 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.7838485996303997 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.24674 -1.00000 + -1.00000 0.00000 + dqin + -0.34754 + eigen coefs + 1.00000 + SCF error = 1.1165739558074710 + + At routine gpmdcov_dm_min; Total charge =-2.78882194E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.78125744E+00 + + SCF iter 3 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 3 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.7812574405507915 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.7812574405507915 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.7812574405507915 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.7812574405507915 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.24674 1.23075 -1.00000 + 1.23075 1.21497 -1.00000 + -1.00000 -1.00000 0.00000 + dqin + -0.34754 -0.34705 + eigen coefs + -76.81151 77.81151 + SCF error = 1.1022571964227903 + + At routine gpmdcov_dm_min; Total charge =-4.23305137E-04 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.77893919E+00 + + SCF iter 4 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 4 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.7789391850593013 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.7789391850593013 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.7789391850593013 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.7789391850593013 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.24674 1.23075 0.00310 -1.00000 + 1.23075 1.21497 0.00310 -1.00000 + 0.00310 0.00310 0.00272 -1.00000 + -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.34754 -0.34705 -0.30977 + eigen coefs +1576.60643********** 21.53132 + SCF error = 5.2119601063489743E-002 + + At routine gpmdcov_dm_min; Total charge =3.45254252E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60371905E+00 + + SCF iter 5 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 5 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.6037190458926409 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.6037190458926409 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.6037190458926409 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.6037190458926409 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.24674 1.23075 0.00310 -0.00297 -1.00000 + 1.23075 1.21497 0.00310 -0.00291 -1.00000 + 0.00310 0.00310 0.00272 0.00191 -1.00000 + -0.00297 -0.00291 0.00191 0.00136 -1.00000 + -1.00000 -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.34754 -0.34705 -0.30977 -0.30979 + eigen coefs + 196.15140-198.69253 0.38812 3.15301 + SCF error = 3.6939351401687748E-002 + + At routine gpmdcov_dm_min; Total charge =1.17315374E-02 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60283503E+00 + + SCF iter 6 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 6 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 1 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 4 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.6028350288692788 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.6028350288692788 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.6028350288692788 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.6028350288692788 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.24674 1.23075 0.00310 -0.00297 -0.00230 -1.00000 + 1.23075 1.21497 0.00310 -0.00291 -0.00227 -1.00000 + 0.00310 0.00310 0.00272 0.00191 0.00006 -1.00000 + -0.00297 -0.00291 0.00191 0.00136 0.00006 -1.00000 + -0.00230 -0.00227 0.00006 0.00006 0.00002 -1.00000 + -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.34754 -0.34705 -0.30977 -0.30979 -0.31034 + eigen coefs +-203.77143 206.40623 1.30673 -5.78504 2.84350 + SCF error = 4.1806895494746624E-003 + + At routine gpmdcov_dm_min; Total charge =1.18539283E-02 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60311456E+00 + + SCF iter 7 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 7 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.6031145579805519 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.6031145579805519 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.6031145579805519 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.6031145579805519 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 1.21497 0.00310 -0.00291 -0.00227 0.00367 -1.00000 + 0.00310 0.00272 0.00191 0.00006 -0.00003 -1.00000 + -0.00291 0.00191 0.00136 0.00006 -0.00004 -1.00000 + -0.00227 0.00006 0.00006 0.00002 0.00000 -1.00000 + 0.00367 -0.00003 -0.00004 0.00000 0.00002 -1.00000 + -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.34705 -0.30977 -0.30979 -0.31034 -0.31055 + eigen coefs + -0.13852 0.22353 1.42045 -29.20422 28.69876 + SCF error = 4.8202699314017303E-003 + + At routine gpmdcov_dm_min; Total charge =2.77479684E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60395482E+00 + + SCF iter 8 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 8 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 2 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 3 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.6039548200467952 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.6039548200467952 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.6039548200467952 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.6039548200467952 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 0.00272 0.00191 0.00006 -0.00003 -0.00000 -1.00000 + 0.00191 0.00136 0.00006 -0.00004 -0.00000 -1.00000 + 0.00006 0.00006 0.00002 0.00000 -0.00000 -1.00000 + -0.00003 -0.00004 0.00000 0.00002 0.00000 -1.00000 + -0.00000 -0.00000 -0.00000 0.00000 0.00000 -1.00000 + -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.30977 -0.30979 -0.31034 -0.31055 -0.31021 + eigen coefs + 0.13105 -0.19728 0.24171 -0.18104 1.00556 + SCF error = 5.4937632692562976E-004 + + At routine gpmdcov_dm_min; Total charge =2.73978990E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60283973E+00 + + + SCF converged within 8 steps ... + SCF error = 5.4937632692562976E-004 + SCF iter 9 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 9 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 1 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 4 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 1.6028397271388151 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = 1.6028397271388151 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = 1.6028397271388151 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = 1.6028397271388151 +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... + + + At routine gpmdcov_DM_Min; Leaving Get charges... + + Performing Pulay mixing scheme ... + coefs + 0.00136 0.00006 -0.00004 -0.00000 0.00000 -1.00000 + 0.00006 0.00002 0.00000 -0.00000 0.00000 -1.00000 + -0.00004 0.00000 0.00002 0.00000 0.00000 -1.00000 + -0.00000 -0.00000 0.00000 0.00000 0.00000 -1.00000 + 0.00000 0.00000 0.00000 0.00000 0.00000 -1.00000 + -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 0.00000 + dqin + -0.30979 -0.31034 -0.31055 -0.31021 -0.31020 + eigen coefs + -0.00051 -0.72063 0.70857 -33.67499 34.68755 + SCF error = 5.1078191859190835E-004 + + At routine gpmdcov_dm_min; Total charge =1.48638826E-03 + + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =1.60281905E+00 + + + At routine gpmdcov_dm_min; Total charge =1.48638826E-03 + + + Total charge of the system (After SCF) = 1.4863882642637272E-003 + + + System charges (After SCF): + H -0.30404068068395979 + O 0.60941303051734097 + H -0.30688034926679203 + H -0.30675131437880138 + O 0.61767464551418683 + H -0.29152968886022879 + H -0.29377779145888971 + O 0.61158678539562916 + H -0.32412544237641905 + H -0.29921583105183730 + O 0.58871007370467832 + H -0.30766741428508576 + H -0.31211776250582968 + O 0.60274094825737756 + H -0.31459617862550887 + H -0.32910721085623279 + O 0.59276664945385205 + H -0.29437354012844691 + H -0.29627799694694223 + O 0.59736778178968453 + H -0.29869359449787808 + H -0.30942278209598789 + O 0.60804769940652792 + H -0.30434668896385381 + H -0.30958077623601937 + O 0.60305393944234409 + H -0.30400276137574783 + H -0.30977392495582140 + O 0.66189390019310224 + H -0.30242346719511920 + H -0.30307934478683052 + O 0.63384558100305721 + H -0.30328329026103590 + H -0.31315463578483999 + O 0.61739606942790726 + H -0.31681897421809224 + H -0.29593612347694004 + O 0.59868467430120198 + H -0.31057980425403259 + H -0.29786122518742647 + O 0.58870390628108282 + H -0.30300151582362850 + H -0.32058070817397077 + O 0.60351971511340075 + H -0.30182981107320384 + H -0.31430448393608446 + O 0.62399508539654103 + H -0.31059100811739604 + H -0.30103892492311735 + O 0.59926368873221492 + H -0.29710147833638673 + H -0.29665121413725509 + O 0.62102275016483888 + H -0.30987205241828614 + H -0.30318565424086341 + O 0.62222151846708573 + H -0.30144841655596594 + H -0.31104586991829225 + O 0.60744017561741359 + H -0.30324751688523321 + H -0.28947419503831096 + O 0.58812474418959393 + H -0.31034257344149307 + H -0.30362517721765481 + O 0.58781683909146554 + H -0.30651656711255465 + H -0.31052223527715528 + O 0.63948610405744200 + H -0.32218881851687781 + H -0.29854442865720637 + O 0.59987619758398492 + H -0.31601466855778571 + H -0.30056123722515427 + O 0.62449319838906880 + H -0.32279382078843583 + H -0.29940055496559120 + O 0.63290303568872541 + H -0.32679049028724150 + H -0.31616589379771320 + O 0.63646118485421299 + H -0.30704231418496469 + H -0.31336139868723661 + O 0.58694466493765252 + H -0.30951416102348295 + H -0.31298646257553830 + O 0.62758364640742270 + H -0.32379585361995961 + H -0.30273802900422092 + O 0.60971589546987781 + H -0.31300270420108883 + H -0.30410652447500680 + O 0.60495438058488227 + H -0.30452595582436276 + H -0.28510645230892068 + O 0.62405780845044023 + H -0.31703139591132423 + H -0.32285395985402637 + O 0.62382814372885376 + H -0.31546933969859986 + H -0.32606135946580816 + O 0.60161420335060001 + H -0.29831131830190144 + H -0.30016937409822086 + O 0.61547778239748185 + H -0.31051264867022199 + H -0.29427026830262132 + O 0.61909068493423969 + H -0.30657558910603955 + H -0.31215776573775861 + O 0.60170907646167371 + H -0.30461125020221125 + H -0.32356825334640377 + O 0.60245659986989719 + H -0.29734820705858417 + H -0.31321164881145624 + O 0.61264038884877559 + H -0.30356650880543801 + H -0.31771332927443813 + O 0.59634886131233833 + H -0.28545355883420481 + H -0.29678381436574430 + O 0.60117501139772211 + H -0.30106298605891885 + H -0.29822703741456952 + O 0.61738988350833734 + H -0.29625854601795154 + H -0.30607173861606835 + O 0.63501178878702991 + H -0.32024274780850698 + H -0.30838071926229926 + O 0.63528064968249698 + H -0.30427307648124297 + H -0.31030513817329181 + O 0.59928582073584180 + H -0.31721095324643844 + H -0.30441297965298791 + O 0.61534083648571192 + H -0.31033183266924647 + H -0.31216379730798965 + O 0.61639059157296272 + H -0.31483674017872909 + H -0.28903815064900562 + O 0.61336751959325764 + H -0.31652997425674750 + H -0.32075002031006106 + O 0.62071037535626172 + H -0.28878624004665504 + H -0.30456834073984251 + O 0.63756244197221812 + H -0.31270263921197100 + H -0.28359834602855138 + O 0.59742451551542419 + H -0.30715967265577770 + H -0.29885520155726308 + O 0.59510133959385281 + H -0.30668934618776011 + H -0.30252352190624465 + O 0.61103390982771921 + H -0.31885097637058707 + H -0.30119102777829726 + O 0.63827487675119954 + H -0.30981299802033341 + H -0.31853063372065549 + O 0.62150795238755741 + H -0.31099922859824669 + H -0.33026309514166102 + O 0.60354166918381635 + H -0.29680610455415657 + H -0.31990116216472297 + O 0.60303633068456020 + H -0.31481086303713202 + H -0.29959367914124913 + O 0.60307802518387965 + H -0.31207055101159203 + H -0.29346967146240449 + O 0.59558146010246649 + H -0.29822342206469871 + H -0.33016620160234744 + O 0.61898816522941502 + H -0.31183419496620546 + H -0.28615581497558579 + O 0.60740218788793998 + H -0.31266188653273852 + H -0.30280655889957764 + O 0.60502547256165851 + H -0.32162787284114830 + H -0.31581488866144064 + O 0.63380723722823418 + H -0.30296296375670434 + H -0.32223612115560835 + O 0.64385500203986601 + H -0.30352323990244978 + H -0.30437602596556190 + O 0.66438779035812057 + H -0.32854515150899910 + H -0.31682384924395135 + O 0.63865809209419000 + H -0.32029220684597232 + H -0.29938377732928162 + O 0.60416717118650631 + H -0.29687564844707398 + H -0.31912155511053342 + O 0.58904304801238128 + H -0.30130487531981426 + H -0.30686540125083739 + O 0.58779661328808308 + H -0.30291464261104178 + H -0.31146610298285143 + O 0.61083195617041397 + H -0.30153833634269001 + H -0.29373010760902851 + O 0.60559223187093336 + H -0.30625648727233556 + H -0.31392854220750910 + O 0.63230583764194093 + H -0.30407748962681880 + H -0.33554252592305722 + O 0.60301692545659502 + H -0.29918921774510510 + H -0.31202437871022992 + O 0.60884284786016263 + H -0.31418349147853430 + H -0.30641175741278986 + O 0.59019768436594122 + H -0.31463502920486675 + H -0.29091795198188630 + O 0.63773642475003212 + H -0.31767238089857563 + H -0.29918445428934931 + O 0.62153220479448446 + H -0.30444810956369672 + H -0.28746239182894395 + O 0.60196263180427900 + H -0.31756495841713756 + H -0.31031494436839763 + O 0.58026477994830794 + H -0.29882115053795688 + H -0.31721568937663974 + O 0.62894854637728514 + H -0.30023046241399726 + H -0.30324043391460703 + O 0.62817997352814658 + H -0.31154252916834957 + H -0.31476680710545946 + O 0.60463578317677280 + H -0.30218250243587846 + H -0.31497070370095509 + O 0.63414161970926575 + H -0.29623504032350567 + H -0.30106774229831701 + O 0.60675706398810092 + H -0.30777794552966747 + H -0.30972743692779492 + O 0.60753608770476830 + H -0.30669892509534147 + H -0.30937515227212720 + O 0.62534443705633291 + H -0.31409298089718746 + H -0.28761185793197619 + O 0.59879533469014001 + H -0.29483016037395859 + H -0.30378702693654824 + O 0.62608259641497632 + H -0.30886706675277292 + H -0.29160442565030026 + O 0.61573031200283501 + H -0.30951948909277494 + H -0.30513075853432892 + O 0.61303558202289365 + H -0.31090540135865141 + H -0.30262076567264584 + O 0.61515479827938191 + H -0.30533953147185422 + H -0.31089612684995283 + O 0.62014308499955084 + H -0.29421370519631823 + H -0.32561800901033783 + O 0.62122099858848234 + H -0.29947459171193219 + H -0.31575492275800637 + O 0.63824267397473089 + H -0.33162464107527695 + H -0.28772021700342293 + O 0.62564419921858139 + H -0.31651167604091118 + H -0.29161064184868724 + O 0.61034417839545840 + H -0.30492764591905458 + H -0.31646775393617105 + O 0.61911229132864631 + H -0.29632895713470153 + H -0.29754276205163338 + O 0.60739745596216954 + H -0.29975228815223554 + H -0.31111289649086871 + O 0.59302161751247839 + H -0.28662724428716491 + H -0.29729848907718592 + O 0.67785335458077745 + H -0.31018035840817471 diff --git a/examples/gpmdk/tests/ref.gpmdcov_FirstCharges b/examples/gpmdk/tests/ref.gpmdcov_FirstCharges new file mode 100644 index 00000000..0698fc4f --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_FirstCharges @@ -0,0 +1,362 @@ + + Building covalency graph ... + + In graph_part .. + prg_metisPartition_test start ... + + total procs = 1 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 4 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 4 localPartExtent = 4 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + MPI rank 1 done with graph_part .. + 1 1 1 265 + 1 2 2 271 + 1 3 3 277 + 1 4 4 262 + Cost per rank = 77750796.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.0000000000000000 + + At routine gpmdcov_Part; Getting CH subsystems ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 2 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 3 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = -6.0000000000000000 +orthop_bml + leaving solver ... + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =9.89084887E-02 + + + Total charge of the system = -1.28518280E+01 + + + Full System charges: + 1 H -0.38287587677689805 + 2 O 0.63744382810575040 + 3 H -0.38541139381519052 + 4 H -0.38651471542141813 + 5 O 0.66237290215892841 + 6 H -0.38504033812318228 + 7 H -0.38540738716704459 + 8 O 0.63787141862758556 + 9 H -0.38597359710200418 + 10 H -0.38757747967414991 + 11 O 0.62807003872116951 + 12 H -0.38682491795099527 + 13 H -0.38410222180602738 + 14 O 0.62461763109826052 + 15 H -0.38698393957444754 + 16 H -0.38440149425428360 + 17 O 0.61164053895860881 + 18 H -0.38569277579158856 + 19 H -0.38434232313365257 + 20 O 0.64463111078439894 + 21 H -0.38462567965123717 + 22 H -0.38427141445758162 + 23 O 0.63690730745572566 + 24 H -0.38291322041581344 + 25 H -0.38381522318335359 + 26 O 0.62992631464225912 + 27 H -0.38350999167534283 + 28 H -0.38880110696455972 + 29 O 0.69204678733768432 + 30 H -0.38488397141888431 + 31 H -0.38614529229822347 + 32 O 0.67075350252162025 + 33 H -0.38404784601704689 + 34 H -0.38452544640896313 + 35 O 0.63230105859271823 + 36 H -0.38575915800819027 + 37 H -0.38602977132028071 + 38 O 0.63606304782821699 + 39 H -0.38378765834477224 + 40 H -0.38403006238803217 + 41 O 0.63075326699853118 + 42 H -0.38514867798428343 + 43 H -0.38398696781303099 + 44 O 0.61860940988109103 + 45 H -0.38379573523016774 + 46 H -0.38482235273610799 + 47 O 0.63819219704432051 + 48 H -0.38509705907999980 + 49 H -0.38317111858896657 + 50 O 0.64160880563308798 + 51 H -0.38267976529147196 + 52 H -0.38373788242832441 + 53 O 0.65612766765774122 + 54 H -0.38563068856926264 + 55 H -0.38797000375734991 + 56 O 0.66220650748357190 + 57 H -0.38652245118314088 + 58 H -0.38483444937967293 + 59 O 0.63561678024539070 + 60 H -0.38513110875621692 + 61 H -0.38313463961007366 + 62 O 0.63369824633151861 + 63 H -0.38450553325596992 + 64 H -0.38432277573572049 + 65 O 0.61941334656292568 + 66 H -0.38377026940288972 + 67 H -0.39115959546148182 + 68 O 0.65138540960355229 + 69 H -0.39150677342225915 + 70 H -0.38789877451764865 + 71 O 0.63429619996013287 + 72 H -0.38904257569339584 + 73 H -0.38425170596187530 + 74 O 0.63725221410264066 + 75 H -0.38355865624878416 + 76 H -0.38685532411767809 + 77 O 0.65232539520473498 + 78 H -0.38798279835701854 + 79 H -0.38411042895466341 + 80 O 0.65773905619248385 + 81 H -0.38505165583432333 + 82 H -0.38198315361624835 + 83 O 0.60533181385956247 + 84 H -0.38273469215827438 + 85 H -0.38489764843445495 + 86 O 0.63465542092936555 + 87 H -0.38749414267057036 + 88 H -0.38522955554360816 + 89 O 0.63690301353478329 + 90 H -0.38346289815765888 + 91 H -0.38262803902409004 + 92 O 0.64189293217961030 + 93 H -0.38304949129675059 + 94 H -0.38337247651731654 + 95 O 0.66069899866248338 + 96 H -0.38716379942533952 + 97 H -0.38788068477271853 + 98 O 0.62264614994751000 + 99 H -0.38687439032101423 + 100 H -0.38611348446694038 + 101 O 0.62438536664749922 + 102 H -0.38461511378639046 + 103 H -0.38526737494943364 + 104 O 0.65027032582021960 + 105 H -0.38644010140223906 + 106 H -0.38409110158860615 + 107 O 0.66003540816961159 + 108 H -0.38413514424952078 + 109 H -0.38547069494616037 + 110 O 0.62418886446238098 + 111 H -0.38295473027726701 + 112 H -0.38119891581465293 + 113 O 0.61937073804837794 + 114 H -0.38105141154062239 + 115 H -0.38762340720661848 + 116 O 0.64114078330534863 + 117 H -0.38590144632591938 + 118 H -0.38425045258382484 + 119 O 0.63118692474923233 + 120 H -0.38301206157437906 + 121 H -0.38494357736923623 + 122 O 0.64077483925598244 + 123 H -0.38491437791989880 + 124 H -0.38555699098850948 + 125 O 0.66441421855959248 + 126 H -0.38399860161497479 + 127 H -0.38695926376822043 + 128 O 0.64622819589657521 + 129 H -0.39086264883170607 + 130 H -0.38781529207117504 + 131 O 0.66359286855500699 + 132 H -0.38635402655560647 + 133 H -0.38422236589019254 + 134 O 0.60880925933512930 + 135 H -0.38522178821507225 + 136 H -0.38318725961900657 + 137 O 0.63999436963939438 + 138 H -0.38503364424014719 + 139 H -0.38444827319404984 + 140 O 0.63162705423999821 + 141 H -0.38685743390000493 + 142 H -0.38438048119938972 + 143 O 0.65037371782720754 + 144 H -0.38556147033059696 + 145 H -0.38659418389048872 + 146 O 0.65153713214770104 + 147 H -0.38348876245634655 + 148 H -0.38572408579796158 + 149 O 0.65773120839195442 + 150 H -0.38539183932618937 + 151 H -0.38397021974045698 + 152 O 0.65009790851119931 + 153 H -0.38441385106118808 + 154 H -0.38389264712556015 + 155 O 0.63320965652431482 + 156 H -0.38432831606532147 + 157 H -0.38458175026261310 + 158 O 0.63542657721683415 + 159 H -0.38511740457510302 + 160 H -0.38757841214223698 + 161 O 0.66869062025316683 + 162 H -0.38617005218926670 + 163 H -0.38695138354532588 + 164 O 0.63727303882707620 + 165 H -0.38574950053513923 + 166 H -0.38575810154876411 + 167 O 0.61333574738949359 + 168 H -0.38255353360042155 + 169 H -0.38671303493857589 + 170 O 0.61045249942596946 + 171 H -0.38365226372453798 + 172 H -0.38547156658637960 + 173 O 0.63302050421687994 + 174 H -0.38364197366170583 + 175 H -0.38236443078989413 + 176 O 0.64653045030051270 + 177 H -0.38254487328351627 + 178 H -0.39676353193869096 + 179 O 0.63111758741108570 + 180 H -0.39116243200286149 + 181 H -0.38166561972706725 + 182 O 0.64780287840810935 + 183 H -0.38260631132431167 + 184 H -0.38476220626925972 + 185 O 0.62211481914551481 + 186 H -0.38381531485769760 + 187 H -0.38799420545609231 + 188 O 0.66171710360223379 + 189 H -0.38640461442453788 + 190 H -0.38215089085834897 + 191 O 0.65257516641836677 + 192 H -0.38317921215679529 + 193 H -0.38798578773062242 + 194 O 0.68209216708275378 + 195 H -0.39714375933507828 + 196 H -0.38547191962063021 + 197 O 0.64336181483855359 + 198 H -0.38651757668281683 + 199 H -0.38276486846342583 + 200 O 0.64958033572627194 + 201 H -0.38085267905139586 + 202 H -0.38338731812501481 + 203 O 0.61119591206047108 + 204 H -0.38306285943501095 + 205 H -0.38299499502653556 + 206 O 0.61515044931978125 + 207 H -0.38356168980813699 + 208 H -0.38388172556888955 + 209 O 0.64211704076982201 + 210 H -0.38408151631059828 + 211 H -0.38500225471000726 + 212 O 0.64873254900008526 + 213 H -0.38290771103533794 + 214 H -0.38757256078164437 + 215 O 0.65786661767194854 + 216 H -0.38758208258894811 + 217 H -0.38833119113424874 + 218 O 0.60323454233107032 + 219 H -0.38236661602340249 + 220 H -0.38433613423468960 + 221 O 0.62409552606179552 + 222 H -0.38737576456748191 + 223 H -0.38293757247369964 + 224 O 0.60869696050420607 + 225 H -0.38326636945256198 + 226 H -0.38564133238539988 + 227 O 0.66763155712650857 + 228 H -0.38687397090242870 + 229 H -0.38646215478652435 + 230 O 0.65757958201066646 + 231 H -0.38385699218984004 + 232 H -0.38190000073041175 + 233 O 0.63852217728445559 + 234 H -0.38513060512857245 + 235 H -0.38428347698072030 + 236 O 0.61350492733636663 + 237 H -0.38468182639188897 + 238 H -0.38539078054099118 + 239 O 0.65334237266432993 + 240 H -0.38581660018478026 + 241 H -0.38529378655888213 + 242 O 0.65313419778199489 + 243 H -0.38520359113499614 + 244 H -0.38398297014967386 + 245 O 0.62491894103860712 + 246 H -0.38491811026513045 + 247 H -0.38439931796994042 + 248 O 0.66442688004916750 + 249 H -0.38430862324754278 + 250 H -0.38348371358233035 + 251 O 0.64037511754996856 + 252 H -0.38474205159406361 + 253 H -0.38266857559150347 + 254 O 0.63505651351883152 + 255 H -0.38467157908611060 + 256 H -0.38676877538739041 + 257 O 0.64637597010395798 + 258 H -0.38609162828711374 + 259 H -0.38388924763936738 + 260 O 0.65844589631668704 + 261 H -0.38281724522328220 + 262 H -0.38481306309278263 + 263 O 0.65662742792211315 + 264 H -0.38469109720402017 + 265 H -0.38448488320095653 + 266 O 0.65909431812519870 + 267 H -0.38563528522701196 + 268 H -0.38493270943074420 + 269 O 0.63772861798929981 + 270 H -0.38399947001702261 + 271 H -0.38360906525456329 + 272 O 0.64715559191300542 + 273 H -0.38481490025647014 + 274 H -0.38383142679871352 + 275 O 0.65983854464200409 + 276 H -0.38603396323834538 + 277 H -0.38308994208739355 + 278 O 0.63603608218827556 + 279 H -0.38692798845397547 + 280 H -0.38864604877136599 + 281 O 0.62828867865037008 + 282 H -0.39580455241394807 + 283 H -0.38853989357943242 + 284 O 0.66894011974030043 + 285 H -0.38835593846051564 + 286 H -0.38412741456803612 + 287 O 0.65787668243718400 + 288 H -0.38435435026896658 + 289 H -0.39067660690435058 + 290 O 0.65260435683509499 + 291 H -0.38921350126075760 + 292 H -0.38337851791964173 + 293 O 0.64876419705635957 + 294 H -0.38255343486336202 + 295 H -0.38456644513480631 + 296 O 0.63383606799740377 + 297 H -0.38380323087890689 + 298 H -0.38730361473968378 + 299 O 0.70697947795857807 + 300 H -0.39086661652135513 diff --git a/examples/gpmdk/tests/ref.gpmdcov_Init b/examples/gpmdk/tests/ref.gpmdcov_Init new file mode 100644 index 00000000..b58baab5 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_Init @@ -0,0 +1,422 @@ + Used mem Before prg_get_covgraph (pid 25769, ppid 25763) = 1717 MiB (1781 MiB) + + Building covalency graph ... + + Used mem After prg_get_covgraph (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + In graph_part .. + prg_metisPartition_test start ... + + total procs = 2 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 2 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 2 localPartExtent = 2 + rank = 1 localPartMin = 3 localPartMax = 4 localPartExtent = 2 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + Time for gpmd_graphpart 2.10000000E+01 ms + MPI rank 1 done with graph_part .. + + At routine gpmdcov_Part; Time for bml_matrix2submatrix_index 0.00000000E+00 ms + + 1 1 1 265 + 1 2 4 262 + Cost per rank = 36594352.000000000 + 2 1 2 271 + 2 2 3 277 + Cost per rank = 41156444.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.12466656056650491 + + At routine gpmdcov_Part; Getting CH subsystems ... + + Used mem Before prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + At routine gpmdcov_Part; Time for prg_get_subsystem 0.00000000E+00 ms + + Used mem After prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + Used mem Before gpmd_InitParts (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + ##################################### + (rank 1) Initializing partition 1 + ##################################### + + Number of atoms in the core = 73 + Number of atoms in the core+halo = 265 + + Time for Build Z for part = 1221 ms + + ##################################### + (rank 1) Initializing partition 4 + ##################################### + + Number of atoms in the core = 75 + Number of atoms in the core+halo = 262 + + Time for Build Z for part = 1203 ms + Used mem After gpmd_InitParts (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = -6.0000000000000000 + Time for Solver = 1315 ms +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = -6.0000000000000000 + Time for Solver = 1422 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =9.89084887E-02 + + + At routine gpmdcov_FirstCharges; MPI rank finished prg_sumIntReduceN for qs =2.00000000E+00 + + + Total charge of the system = -1.28518280E+01 + + + Full System charges: + 1 H -0.38287587677689805 + 2 O 0.63744382810575040 + 3 H -0.38541139381519052 + 4 H -0.38651471542141813 + 5 O 0.66237290215892841 + 6 H -0.38504033812318228 + 7 H -0.38540738716704459 + 8 O 0.63787141862758556 + 9 H -0.38597359710200418 + 10 H -0.38757747967414991 + 11 O 0.62807003872116951 + 12 H -0.38682491795099527 + 13 H -0.38410222180602738 + 14 O 0.62461763109826052 + 15 H -0.38698393957444754 + 16 H -0.38440149425428360 + 17 O 0.61164053895860881 + 18 H -0.38569277579158856 + 19 H -0.38434232313365257 + 20 O 0.64463111078439894 + 21 H -0.38462567965123717 + 22 H -0.38427141445758162 + 23 O 0.63690730745572566 + 24 H -0.38291322041581344 + 25 H -0.38381522318335359 + 26 O 0.62992631464225912 + 27 H -0.38350999167534283 + 28 H -0.38880110696455972 + 29 O 0.69204678733768432 + 30 H -0.38488397141888431 + 31 H -0.38614529229822347 + 32 O 0.67075350252162025 + 33 H -0.38404784601704689 + 34 H -0.38452544640896313 + 35 O 0.63230105859271823 + 36 H -0.38575915800819027 + 37 H -0.38602977132028071 + 38 O 0.63606304782821699 + 39 H -0.38378765834477224 + 40 H -0.38403006238803217 + 41 O 0.63075326699853118 + 42 H -0.38514867798428343 + 43 H -0.38398696781303099 + 44 O 0.61860940988109103 + 45 H -0.38379573523016774 + 46 H -0.38482235273610799 + 47 O 0.63819219704432051 + 48 H -0.38509705907999980 + 49 H -0.38317111858896657 + 50 O 0.64160880563308798 + 51 H -0.38267976529147196 + 52 H -0.38373788242832441 + 53 O 0.65612766765774122 + 54 H -0.38563068856926264 + 55 H -0.38797000375734991 + 56 O 0.66220650748357190 + 57 H -0.38652245118314088 + 58 H -0.38483444937967293 + 59 O 0.63561678024539070 + 60 H -0.38513110875621692 + 61 H -0.38313463961007366 + 62 O 0.63369824633151861 + 63 H -0.38450553325596992 + 64 H -0.38432277573572049 + 65 O 0.61941334656292568 + 66 H -0.38377026940288972 + 67 H -0.39115959546148182 + 68 O 0.65138540960355229 + 69 H -0.39150677342225915 + 70 H -0.38789877451764865 + 71 O 0.63429619996013287 + 72 H -0.38904257569339584 + 73 H -0.38425170596187530 + 74 O 0.63725221410264066 + 75 H -0.38355865624878416 + 76 H -0.38685532411767809 + 77 O 0.65232539520473498 + 78 H -0.38798279835701854 + 79 H -0.38411042895466341 + 80 O 0.65773905619248385 + 81 H -0.38505165583432333 + 82 H -0.38198315361624835 + 83 O 0.60533181385956247 + 84 H -0.38273469215827438 + 85 H -0.38489764843445495 + 86 O 0.63465542092936555 + 87 H -0.38749414267057036 + 88 H -0.38522955554360816 + 89 O 0.63690301353478329 + 90 H -0.38346289815765888 + 91 H -0.38262803902409004 + 92 O 0.64189293217961030 + 93 H -0.38304949129675059 + 94 H -0.38337247651731654 + 95 O 0.66069899866248338 + 96 H -0.38716379942533952 + 97 H -0.38788068477271853 + 98 O 0.62264614994751000 + 99 H -0.38687439032101423 + 100 H -0.38611348446694038 + 101 O 0.62438536664749922 + 102 H -0.38461511378639046 + 103 H -0.38526737494943364 + 104 O 0.65027032582021960 + 105 H -0.38644010140223906 + 106 H -0.38409110158860615 + 107 O 0.66003540816961159 + 108 H -0.38413514424952078 + 109 H -0.38547069494616037 + 110 O 0.62418886446238098 + 111 H -0.38295473027726701 + 112 H -0.38119891581465293 + 113 O 0.61937073804837794 + 114 H -0.38105141154062239 + 115 H -0.38762340720661848 + 116 O 0.64114078330534863 + 117 H -0.38590144632591938 + 118 H -0.38425045258382484 + 119 O 0.63118692474923233 + 120 H -0.38301206157437906 + 121 H -0.38494357736923623 + 122 O 0.64077483925598244 + 123 H -0.38491437791989880 + 124 H -0.38555699098850948 + 125 O 0.66441421855959248 + 126 H -0.38399860161497479 + 127 H -0.38695926376822043 + 128 O 0.64622819589657521 + 129 H -0.39086264883170607 + 130 H -0.38781529207117504 + 131 O 0.66359286855500699 + 132 H -0.38635402655560647 + 133 H -0.38422236589019254 + 134 O 0.60880925933512930 + 135 H -0.38522178821507225 + 136 H -0.38318725961900657 + 137 O 0.63999436963939438 + 138 H -0.38503364424014719 + 139 H -0.38444827319404984 + 140 O 0.63162705423999821 + 141 H -0.38685743390000493 + 142 H -0.38438048119938972 + 143 O 0.65037371782720754 + 144 H -0.38556147033059696 + 145 H -0.38659418389048872 + 146 O 0.65153713214770104 + 147 H -0.38348876245634655 + 148 H -0.38572408579796158 + 149 O 0.65773120839195442 + 150 H -0.38539183932618937 + 151 H -0.38397021974045698 + 152 O 0.65009790851119931 + 153 H -0.38441385106118808 + 154 H -0.38389264712556015 + 155 O 0.63320965652431482 + 156 H -0.38432831606532147 + 157 H -0.38458175026261310 + 158 O 0.63542657721683415 + 159 H -0.38511740457510302 + 160 H -0.38757841214223698 + 161 O 0.66869062025316683 + 162 H -0.38617005218926670 + 163 H -0.38695138354532588 + 164 O 0.63727303882707620 + 165 H -0.38574950053513923 + 166 H -0.38575810154876411 + 167 O 0.61333574738949359 + 168 H -0.38255353360042155 + 169 H -0.38671303493857589 + 170 O 0.61045249942596946 + 171 H -0.38365226372453798 + 172 H -0.38547156658637960 + 173 O 0.63302050421687994 + 174 H -0.38364197366170583 + 175 H -0.38236443078989413 + 176 O 0.64653045030051270 + 177 H -0.38254487328351627 + 178 H -0.39676353193869096 + 179 O 0.63111758741108570 + 180 H -0.39116243200286149 + 181 H -0.38166561972706725 + 182 O 0.64780287840810935 + 183 H -0.38260631132431167 + 184 H -0.38476220626925972 + 185 O 0.62211481914551481 + 186 H -0.38381531485769760 + 187 H -0.38799420545609231 + 188 O 0.66171710360223379 + 189 H -0.38640461442453788 + 190 H -0.38215089085834897 + 191 O 0.65257516641836677 + 192 H -0.38317921215679529 + 193 H -0.38798578773062242 + 194 O 0.68209216708275378 + 195 H -0.39714375933507828 + 196 H -0.38547191962063021 + 197 O 0.64336181483855359 + 198 H -0.38651757668281683 + 199 H -0.38276486846342583 + 200 O 0.64958033572627194 + 201 H -0.38085267905139586 + 202 H -0.38338731812501481 + 203 O 0.61119591206047108 + 204 H -0.38306285943501095 + 205 H -0.38299499502653556 + 206 O 0.61515044931978125 + 207 H -0.38356168980813699 + 208 H -0.38388172556888955 + 209 O 0.64211704076982201 + 210 H -0.38408151631059828 + 211 H -0.38500225471000726 + 212 O 0.64873254900008526 + 213 H -0.38290771103533794 + 214 H -0.38757256078164437 + 215 O 0.65786661767194854 + 216 H -0.38758208258894811 + 217 H -0.38833119113424874 + 218 O 0.60323454233107032 + 219 H -0.38236661602340249 + 220 H -0.38433613423468960 + 221 O 0.62409552606179552 + 222 H -0.38737576456748191 + 223 H -0.38293757247369964 + 224 O 0.60869696050420607 + 225 H -0.38326636945256198 + 226 H -0.38564133238539988 + 227 O 0.66763155712650857 + 228 H -0.38687397090242870 + 229 H -0.38646215478652435 + 230 O 0.65757958201066646 + 231 H -0.38385699218984004 + 232 H -0.38190000073041175 + 233 O 0.63852217728445559 + 234 H -0.38513060512857245 + 235 H -0.38428347698072030 + 236 O 0.61350492733636663 + 237 H -0.38468182639188897 + 238 H -0.38539078054099118 + 239 O 0.65334237266432993 + 240 H -0.38581660018478026 + 241 H -0.38529378655888213 + 242 O 0.65313419778199489 + 243 H -0.38520359113499614 + 244 H -0.38398297014967386 + 245 O 0.62491894103860712 + 246 H -0.38491811026513045 + 247 H -0.38439931796994042 + 248 O 0.66442688004916750 + 249 H -0.38430862324754278 + 250 H -0.38348371358233035 + 251 O 0.64037511754996856 + 252 H -0.38474205159406361 + 253 H -0.38266857559150347 + 254 O 0.63505651351883152 + 255 H -0.38467157908611060 + 256 H -0.38676877538739041 + 257 O 0.64637597010395798 + 258 H -0.38609162828711374 + 259 H -0.38388924763936738 + 260 O 0.65844589631668704 + 261 H -0.38281724522328220 + 262 H -0.38481306309278263 + 263 O 0.65662742792211315 + 264 H -0.38469109720402017 + 265 H -0.38448488320095653 + 266 O 0.65909431812519870 + 267 H -0.38563528522701196 + 268 H -0.38493270943074420 + 269 O 0.63772861798929981 + 270 H -0.38399947001702261 + 271 H -0.38360906525456329 + 272 O 0.64715559191300542 + 273 H -0.38481490025647014 + 274 H -0.38383142679871352 + 275 O 0.65983854464200409 + 276 H -0.38603396323834538 + 277 H -0.38308994208739355 + 278 O 0.63603608218827556 + 279 H -0.38692798845397547 + 280 H -0.38864604877136599 + 281 O 0.62828867865037008 + 282 H -0.39580455241394807 + 283 H -0.38853989357943242 + 284 O 0.66894011974030043 + 285 H -0.38835593846051564 + 286 H -0.38412741456803612 + 287 O 0.65787668243718400 + 288 H -0.38435435026896658 + 289 H -0.39067660690435058 + 290 O 0.65260435683509499 + 291 H -0.38921350126075760 + 292 H -0.38337851791964173 + 293 O 0.64876419705635957 + 294 H -0.38255343486336202 + 295 H -0.38456644513480631 + 296 O 0.63383606799740377 + 297 H -0.38380323087890689 + 298 H -0.38730361473968378 + 299 O 0.70697947795857807 + 300 H -0.39086661652135513 + Used mem After gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_DM_Min (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + SCF iter 1 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 1 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 13 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 12 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 9.8908488652095117E-002 + Time for Solver = 699 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... diff --git a/examples/gpmdk/tests/ref.gpmdcov_InitParts b/examples/gpmdk/tests/ref.gpmdcov_InitParts new file mode 100644 index 00000000..3d35d7d4 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_InitParts @@ -0,0 +1,28 @@ + + Building covalency graph ... + + In graph_part .. + prg_metisPartition_test start ... + + total procs = 1 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 4 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 4 localPartExtent = 4 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + MPI rank 1 done with graph_part .. + 1 1 1 265 + 1 2 2 271 + 1 3 3 277 + 1 4 4 262 + Cost per rank = 77750796.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.0000000000000000 + + At routine gpmdcov_Part; Getting CH subsystems ... + diff --git a/examples/gpmdk/tests/ref.gpmdcov_MDloop b/examples/gpmdk/tests/ref.gpmdcov_MDloop new file mode 100644 index 00000000..b58baab5 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_MDloop @@ -0,0 +1,422 @@ + Used mem Before prg_get_covgraph (pid 25769, ppid 25763) = 1717 MiB (1781 MiB) + + Building covalency graph ... + + Used mem After prg_get_covgraph (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + In graph_part .. + prg_metisPartition_test start ... + + total procs = 2 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 2 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 2 localPartExtent = 2 + rank = 1 localPartMin = 3 localPartMax = 4 localPartExtent = 2 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + Time for gpmd_graphpart 2.10000000E+01 ms + MPI rank 1 done with graph_part .. + + At routine gpmdcov_Part; Time for bml_matrix2submatrix_index 0.00000000E+00 ms + + 1 1 1 265 + 1 2 4 262 + Cost per rank = 36594352.000000000 + 2 1 2 271 + 2 2 3 277 + Cost per rank = 41156444.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.12466656056650491 + + At routine gpmdcov_Part; Getting CH subsystems ... + + Used mem Before prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + At routine gpmdcov_Part; Time for prg_get_subsystem 0.00000000E+00 ms + + Used mem After prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + Used mem Before gpmd_InitParts (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + ##################################### + (rank 1) Initializing partition 1 + ##################################### + + Number of atoms in the core = 73 + Number of atoms in the core+halo = 265 + + Time for Build Z for part = 1221 ms + + ##################################### + (rank 1) Initializing partition 4 + ##################################### + + Number of atoms in the core = 75 + Number of atoms in the core+halo = 262 + + Time for Build Z for part = 1203 ms + Used mem After gpmd_InitParts (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = -6.0000000000000000 + Time for Solver = 1315 ms +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = -6.0000000000000000 + Time for Solver = 1422 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =9.89084887E-02 + + + At routine gpmdcov_FirstCharges; MPI rank finished prg_sumIntReduceN for qs =2.00000000E+00 + + + Total charge of the system = -1.28518280E+01 + + + Full System charges: + 1 H -0.38287587677689805 + 2 O 0.63744382810575040 + 3 H -0.38541139381519052 + 4 H -0.38651471542141813 + 5 O 0.66237290215892841 + 6 H -0.38504033812318228 + 7 H -0.38540738716704459 + 8 O 0.63787141862758556 + 9 H -0.38597359710200418 + 10 H -0.38757747967414991 + 11 O 0.62807003872116951 + 12 H -0.38682491795099527 + 13 H -0.38410222180602738 + 14 O 0.62461763109826052 + 15 H -0.38698393957444754 + 16 H -0.38440149425428360 + 17 O 0.61164053895860881 + 18 H -0.38569277579158856 + 19 H -0.38434232313365257 + 20 O 0.64463111078439894 + 21 H -0.38462567965123717 + 22 H -0.38427141445758162 + 23 O 0.63690730745572566 + 24 H -0.38291322041581344 + 25 H -0.38381522318335359 + 26 O 0.62992631464225912 + 27 H -0.38350999167534283 + 28 H -0.38880110696455972 + 29 O 0.69204678733768432 + 30 H -0.38488397141888431 + 31 H -0.38614529229822347 + 32 O 0.67075350252162025 + 33 H -0.38404784601704689 + 34 H -0.38452544640896313 + 35 O 0.63230105859271823 + 36 H -0.38575915800819027 + 37 H -0.38602977132028071 + 38 O 0.63606304782821699 + 39 H -0.38378765834477224 + 40 H -0.38403006238803217 + 41 O 0.63075326699853118 + 42 H -0.38514867798428343 + 43 H -0.38398696781303099 + 44 O 0.61860940988109103 + 45 H -0.38379573523016774 + 46 H -0.38482235273610799 + 47 O 0.63819219704432051 + 48 H -0.38509705907999980 + 49 H -0.38317111858896657 + 50 O 0.64160880563308798 + 51 H -0.38267976529147196 + 52 H -0.38373788242832441 + 53 O 0.65612766765774122 + 54 H -0.38563068856926264 + 55 H -0.38797000375734991 + 56 O 0.66220650748357190 + 57 H -0.38652245118314088 + 58 H -0.38483444937967293 + 59 O 0.63561678024539070 + 60 H -0.38513110875621692 + 61 H -0.38313463961007366 + 62 O 0.63369824633151861 + 63 H -0.38450553325596992 + 64 H -0.38432277573572049 + 65 O 0.61941334656292568 + 66 H -0.38377026940288972 + 67 H -0.39115959546148182 + 68 O 0.65138540960355229 + 69 H -0.39150677342225915 + 70 H -0.38789877451764865 + 71 O 0.63429619996013287 + 72 H -0.38904257569339584 + 73 H -0.38425170596187530 + 74 O 0.63725221410264066 + 75 H -0.38355865624878416 + 76 H -0.38685532411767809 + 77 O 0.65232539520473498 + 78 H -0.38798279835701854 + 79 H -0.38411042895466341 + 80 O 0.65773905619248385 + 81 H -0.38505165583432333 + 82 H -0.38198315361624835 + 83 O 0.60533181385956247 + 84 H -0.38273469215827438 + 85 H -0.38489764843445495 + 86 O 0.63465542092936555 + 87 H -0.38749414267057036 + 88 H -0.38522955554360816 + 89 O 0.63690301353478329 + 90 H -0.38346289815765888 + 91 H -0.38262803902409004 + 92 O 0.64189293217961030 + 93 H -0.38304949129675059 + 94 H -0.38337247651731654 + 95 O 0.66069899866248338 + 96 H -0.38716379942533952 + 97 H -0.38788068477271853 + 98 O 0.62264614994751000 + 99 H -0.38687439032101423 + 100 H -0.38611348446694038 + 101 O 0.62438536664749922 + 102 H -0.38461511378639046 + 103 H -0.38526737494943364 + 104 O 0.65027032582021960 + 105 H -0.38644010140223906 + 106 H -0.38409110158860615 + 107 O 0.66003540816961159 + 108 H -0.38413514424952078 + 109 H -0.38547069494616037 + 110 O 0.62418886446238098 + 111 H -0.38295473027726701 + 112 H -0.38119891581465293 + 113 O 0.61937073804837794 + 114 H -0.38105141154062239 + 115 H -0.38762340720661848 + 116 O 0.64114078330534863 + 117 H -0.38590144632591938 + 118 H -0.38425045258382484 + 119 O 0.63118692474923233 + 120 H -0.38301206157437906 + 121 H -0.38494357736923623 + 122 O 0.64077483925598244 + 123 H -0.38491437791989880 + 124 H -0.38555699098850948 + 125 O 0.66441421855959248 + 126 H -0.38399860161497479 + 127 H -0.38695926376822043 + 128 O 0.64622819589657521 + 129 H -0.39086264883170607 + 130 H -0.38781529207117504 + 131 O 0.66359286855500699 + 132 H -0.38635402655560647 + 133 H -0.38422236589019254 + 134 O 0.60880925933512930 + 135 H -0.38522178821507225 + 136 H -0.38318725961900657 + 137 O 0.63999436963939438 + 138 H -0.38503364424014719 + 139 H -0.38444827319404984 + 140 O 0.63162705423999821 + 141 H -0.38685743390000493 + 142 H -0.38438048119938972 + 143 O 0.65037371782720754 + 144 H -0.38556147033059696 + 145 H -0.38659418389048872 + 146 O 0.65153713214770104 + 147 H -0.38348876245634655 + 148 H -0.38572408579796158 + 149 O 0.65773120839195442 + 150 H -0.38539183932618937 + 151 H -0.38397021974045698 + 152 O 0.65009790851119931 + 153 H -0.38441385106118808 + 154 H -0.38389264712556015 + 155 O 0.63320965652431482 + 156 H -0.38432831606532147 + 157 H -0.38458175026261310 + 158 O 0.63542657721683415 + 159 H -0.38511740457510302 + 160 H -0.38757841214223698 + 161 O 0.66869062025316683 + 162 H -0.38617005218926670 + 163 H -0.38695138354532588 + 164 O 0.63727303882707620 + 165 H -0.38574950053513923 + 166 H -0.38575810154876411 + 167 O 0.61333574738949359 + 168 H -0.38255353360042155 + 169 H -0.38671303493857589 + 170 O 0.61045249942596946 + 171 H -0.38365226372453798 + 172 H -0.38547156658637960 + 173 O 0.63302050421687994 + 174 H -0.38364197366170583 + 175 H -0.38236443078989413 + 176 O 0.64653045030051270 + 177 H -0.38254487328351627 + 178 H -0.39676353193869096 + 179 O 0.63111758741108570 + 180 H -0.39116243200286149 + 181 H -0.38166561972706725 + 182 O 0.64780287840810935 + 183 H -0.38260631132431167 + 184 H -0.38476220626925972 + 185 O 0.62211481914551481 + 186 H -0.38381531485769760 + 187 H -0.38799420545609231 + 188 O 0.66171710360223379 + 189 H -0.38640461442453788 + 190 H -0.38215089085834897 + 191 O 0.65257516641836677 + 192 H -0.38317921215679529 + 193 H -0.38798578773062242 + 194 O 0.68209216708275378 + 195 H -0.39714375933507828 + 196 H -0.38547191962063021 + 197 O 0.64336181483855359 + 198 H -0.38651757668281683 + 199 H -0.38276486846342583 + 200 O 0.64958033572627194 + 201 H -0.38085267905139586 + 202 H -0.38338731812501481 + 203 O 0.61119591206047108 + 204 H -0.38306285943501095 + 205 H -0.38299499502653556 + 206 O 0.61515044931978125 + 207 H -0.38356168980813699 + 208 H -0.38388172556888955 + 209 O 0.64211704076982201 + 210 H -0.38408151631059828 + 211 H -0.38500225471000726 + 212 O 0.64873254900008526 + 213 H -0.38290771103533794 + 214 H -0.38757256078164437 + 215 O 0.65786661767194854 + 216 H -0.38758208258894811 + 217 H -0.38833119113424874 + 218 O 0.60323454233107032 + 219 H -0.38236661602340249 + 220 H -0.38433613423468960 + 221 O 0.62409552606179552 + 222 H -0.38737576456748191 + 223 H -0.38293757247369964 + 224 O 0.60869696050420607 + 225 H -0.38326636945256198 + 226 H -0.38564133238539988 + 227 O 0.66763155712650857 + 228 H -0.38687397090242870 + 229 H -0.38646215478652435 + 230 O 0.65757958201066646 + 231 H -0.38385699218984004 + 232 H -0.38190000073041175 + 233 O 0.63852217728445559 + 234 H -0.38513060512857245 + 235 H -0.38428347698072030 + 236 O 0.61350492733636663 + 237 H -0.38468182639188897 + 238 H -0.38539078054099118 + 239 O 0.65334237266432993 + 240 H -0.38581660018478026 + 241 H -0.38529378655888213 + 242 O 0.65313419778199489 + 243 H -0.38520359113499614 + 244 H -0.38398297014967386 + 245 O 0.62491894103860712 + 246 H -0.38491811026513045 + 247 H -0.38439931796994042 + 248 O 0.66442688004916750 + 249 H -0.38430862324754278 + 250 H -0.38348371358233035 + 251 O 0.64037511754996856 + 252 H -0.38474205159406361 + 253 H -0.38266857559150347 + 254 O 0.63505651351883152 + 255 H -0.38467157908611060 + 256 H -0.38676877538739041 + 257 O 0.64637597010395798 + 258 H -0.38609162828711374 + 259 H -0.38388924763936738 + 260 O 0.65844589631668704 + 261 H -0.38281724522328220 + 262 H -0.38481306309278263 + 263 O 0.65662742792211315 + 264 H -0.38469109720402017 + 265 H -0.38448488320095653 + 266 O 0.65909431812519870 + 267 H -0.38563528522701196 + 268 H -0.38493270943074420 + 269 O 0.63772861798929981 + 270 H -0.38399947001702261 + 271 H -0.38360906525456329 + 272 O 0.64715559191300542 + 273 H -0.38481490025647014 + 274 H -0.38383142679871352 + 275 O 0.65983854464200409 + 276 H -0.38603396323834538 + 277 H -0.38308994208739355 + 278 O 0.63603608218827556 + 279 H -0.38692798845397547 + 280 H -0.38864604877136599 + 281 O 0.62828867865037008 + 282 H -0.39580455241394807 + 283 H -0.38853989357943242 + 284 O 0.66894011974030043 + 285 H -0.38835593846051564 + 286 H -0.38412741456803612 + 287 O 0.65787668243718400 + 288 H -0.38435435026896658 + 289 H -0.39067660690435058 + 290 O 0.65260435683509499 + 291 H -0.38921350126075760 + 292 H -0.38337851791964173 + 293 O 0.64876419705635957 + 294 H -0.38255343486336202 + 295 H -0.38456644513480631 + 296 O 0.63383606799740377 + 297 H -0.38380323087890689 + 298 H -0.38730361473968378 + 299 O 0.70697947795857807 + 300 H -0.39086661652135513 + Used mem After gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_DM_Min (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + SCF iter 1 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 1 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 13 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 12 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 9.8908488652095117E-002 + Time for Solver = 699 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... diff --git a/examples/gpmdk/tests/ref.gpmdcov_Part b/examples/gpmdk/tests/ref.gpmdcov_Part new file mode 100644 index 00000000..3d35d7d4 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_Part @@ -0,0 +1,28 @@ + + Building covalency graph ... + + In graph_part .. + prg_metisPartition_test start ... + + total procs = 1 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 4 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 4 localPartExtent = 4 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + MPI rank 1 done with graph_part .. + 1 1 1 265 + 1 2 2 271 + 1 3 3 277 + 1 4 4 262 + Cost per rank = 77750796.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.0000000000000000 + + At routine gpmdcov_Part; Getting CH subsystems ... + diff --git a/examples/gpmdk/tests/ref.gpmdcov_PrepareMD b/examples/gpmdk/tests/ref.gpmdcov_PrepareMD new file mode 100644 index 00000000..b58baab5 --- /dev/null +++ b/examples/gpmdk/tests/ref.gpmdcov_PrepareMD @@ -0,0 +1,422 @@ + Used mem Before prg_get_covgraph (pid 25769, ppid 25763) = 1717 MiB (1781 MiB) + + Building covalency graph ... + + Used mem After prg_get_covgraph (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + In graph_part .. + prg_metisPartition_test start ... + + total procs = 2 + total nodes = 300 + total nodes2 = 300 + total parts = 4 + local parts = 2 + + globalPartMin = 1 globalPartMax = 4 globalPartExtent = 4 + rank = 0 localPartMin = 1 localPartMax = 2 localPartExtent = 2 + rank = 1 localPartMin = 3 localPartMax = 4 localPartExtent = 2 + + The number of nodes in the graph is: 300 300 1 4 1 + Cost of METIS 10364163.000000000 140.00000000000000 6.0000000000000000 + Time for gpmd_graphpart 2.10000000E+01 ms + MPI rank 1 done with graph_part .. + + At routine gpmdcov_Part; Time for bml_matrix2submatrix_index 0.00000000E+00 ms + + 1 1 1 265 + 1 2 4 262 + Cost per rank = 36594352.000000000 + 2 1 2 271 + 2 2 3 277 + Cost per rank = 41156444.000000000 + The following is a measure of the asymmetry + DeltaCostPerrank/CostPerrankMin = 0.12466656056650491 + + At routine gpmdcov_Part; Getting CH subsystems ... + + Used mem Before prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + At routine gpmdcov_Part; Time for prg_get_subsystem 0.00000000E+00 ms + + Used mem After prg_get_subsystem (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + Used mem Before gpmd_InitParts (pid 25769, ppid 25763) = 1718 MiB (1781 MiB) + + ##################################### + (rank 1) Initializing partition 1 + ##################################### + + Number of atoms in the core = 73 + Number of atoms in the core+halo = 265 + + Time for Build Z for part = 1221 ms + + ##################################### + (rank 1) Initializing partition 4 + ##################################### + + Number of atoms in the core = 75 + Number of atoms in the core+halo = 262 + + Time for Build Z for part = 1203 ms + Used mem After gpmd_InitParts (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = -6.0000000000000000 + Time for Solver = 1315 ms +orthop_bml + leaving solver ... + starting solver ... + In get_density_t_efd ... + ipt = 4 Ef = -6.0000000000000000 + Time for Solver = 1422 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_getmu; In gpmdcov_getmu ... + + + At routine gpmdcov_musearch; In gpmdcov_musearch ... + + + At routine gpmdcov_muFromParts; Chemical potential (Mu or Ef) =9.89084887E-02 + + + At routine gpmdcov_FirstCharges; MPI rank finished prg_sumIntReduceN for qs =2.00000000E+00 + + + Total charge of the system = -1.28518280E+01 + + + Full System charges: + 1 H -0.38287587677689805 + 2 O 0.63744382810575040 + 3 H -0.38541139381519052 + 4 H -0.38651471542141813 + 5 O 0.66237290215892841 + 6 H -0.38504033812318228 + 7 H -0.38540738716704459 + 8 O 0.63787141862758556 + 9 H -0.38597359710200418 + 10 H -0.38757747967414991 + 11 O 0.62807003872116951 + 12 H -0.38682491795099527 + 13 H -0.38410222180602738 + 14 O 0.62461763109826052 + 15 H -0.38698393957444754 + 16 H -0.38440149425428360 + 17 O 0.61164053895860881 + 18 H -0.38569277579158856 + 19 H -0.38434232313365257 + 20 O 0.64463111078439894 + 21 H -0.38462567965123717 + 22 H -0.38427141445758162 + 23 O 0.63690730745572566 + 24 H -0.38291322041581344 + 25 H -0.38381522318335359 + 26 O 0.62992631464225912 + 27 H -0.38350999167534283 + 28 H -0.38880110696455972 + 29 O 0.69204678733768432 + 30 H -0.38488397141888431 + 31 H -0.38614529229822347 + 32 O 0.67075350252162025 + 33 H -0.38404784601704689 + 34 H -0.38452544640896313 + 35 O 0.63230105859271823 + 36 H -0.38575915800819027 + 37 H -0.38602977132028071 + 38 O 0.63606304782821699 + 39 H -0.38378765834477224 + 40 H -0.38403006238803217 + 41 O 0.63075326699853118 + 42 H -0.38514867798428343 + 43 H -0.38398696781303099 + 44 O 0.61860940988109103 + 45 H -0.38379573523016774 + 46 H -0.38482235273610799 + 47 O 0.63819219704432051 + 48 H -0.38509705907999980 + 49 H -0.38317111858896657 + 50 O 0.64160880563308798 + 51 H -0.38267976529147196 + 52 H -0.38373788242832441 + 53 O 0.65612766765774122 + 54 H -0.38563068856926264 + 55 H -0.38797000375734991 + 56 O 0.66220650748357190 + 57 H -0.38652245118314088 + 58 H -0.38483444937967293 + 59 O 0.63561678024539070 + 60 H -0.38513110875621692 + 61 H -0.38313463961007366 + 62 O 0.63369824633151861 + 63 H -0.38450553325596992 + 64 H -0.38432277573572049 + 65 O 0.61941334656292568 + 66 H -0.38377026940288972 + 67 H -0.39115959546148182 + 68 O 0.65138540960355229 + 69 H -0.39150677342225915 + 70 H -0.38789877451764865 + 71 O 0.63429619996013287 + 72 H -0.38904257569339584 + 73 H -0.38425170596187530 + 74 O 0.63725221410264066 + 75 H -0.38355865624878416 + 76 H -0.38685532411767809 + 77 O 0.65232539520473498 + 78 H -0.38798279835701854 + 79 H -0.38411042895466341 + 80 O 0.65773905619248385 + 81 H -0.38505165583432333 + 82 H -0.38198315361624835 + 83 O 0.60533181385956247 + 84 H -0.38273469215827438 + 85 H -0.38489764843445495 + 86 O 0.63465542092936555 + 87 H -0.38749414267057036 + 88 H -0.38522955554360816 + 89 O 0.63690301353478329 + 90 H -0.38346289815765888 + 91 H -0.38262803902409004 + 92 O 0.64189293217961030 + 93 H -0.38304949129675059 + 94 H -0.38337247651731654 + 95 O 0.66069899866248338 + 96 H -0.38716379942533952 + 97 H -0.38788068477271853 + 98 O 0.62264614994751000 + 99 H -0.38687439032101423 + 100 H -0.38611348446694038 + 101 O 0.62438536664749922 + 102 H -0.38461511378639046 + 103 H -0.38526737494943364 + 104 O 0.65027032582021960 + 105 H -0.38644010140223906 + 106 H -0.38409110158860615 + 107 O 0.66003540816961159 + 108 H -0.38413514424952078 + 109 H -0.38547069494616037 + 110 O 0.62418886446238098 + 111 H -0.38295473027726701 + 112 H -0.38119891581465293 + 113 O 0.61937073804837794 + 114 H -0.38105141154062239 + 115 H -0.38762340720661848 + 116 O 0.64114078330534863 + 117 H -0.38590144632591938 + 118 H -0.38425045258382484 + 119 O 0.63118692474923233 + 120 H -0.38301206157437906 + 121 H -0.38494357736923623 + 122 O 0.64077483925598244 + 123 H -0.38491437791989880 + 124 H -0.38555699098850948 + 125 O 0.66441421855959248 + 126 H -0.38399860161497479 + 127 H -0.38695926376822043 + 128 O 0.64622819589657521 + 129 H -0.39086264883170607 + 130 H -0.38781529207117504 + 131 O 0.66359286855500699 + 132 H -0.38635402655560647 + 133 H -0.38422236589019254 + 134 O 0.60880925933512930 + 135 H -0.38522178821507225 + 136 H -0.38318725961900657 + 137 O 0.63999436963939438 + 138 H -0.38503364424014719 + 139 H -0.38444827319404984 + 140 O 0.63162705423999821 + 141 H -0.38685743390000493 + 142 H -0.38438048119938972 + 143 O 0.65037371782720754 + 144 H -0.38556147033059696 + 145 H -0.38659418389048872 + 146 O 0.65153713214770104 + 147 H -0.38348876245634655 + 148 H -0.38572408579796158 + 149 O 0.65773120839195442 + 150 H -0.38539183932618937 + 151 H -0.38397021974045698 + 152 O 0.65009790851119931 + 153 H -0.38441385106118808 + 154 H -0.38389264712556015 + 155 O 0.63320965652431482 + 156 H -0.38432831606532147 + 157 H -0.38458175026261310 + 158 O 0.63542657721683415 + 159 H -0.38511740457510302 + 160 H -0.38757841214223698 + 161 O 0.66869062025316683 + 162 H -0.38617005218926670 + 163 H -0.38695138354532588 + 164 O 0.63727303882707620 + 165 H -0.38574950053513923 + 166 H -0.38575810154876411 + 167 O 0.61333574738949359 + 168 H -0.38255353360042155 + 169 H -0.38671303493857589 + 170 O 0.61045249942596946 + 171 H -0.38365226372453798 + 172 H -0.38547156658637960 + 173 O 0.63302050421687994 + 174 H -0.38364197366170583 + 175 H -0.38236443078989413 + 176 O 0.64653045030051270 + 177 H -0.38254487328351627 + 178 H -0.39676353193869096 + 179 O 0.63111758741108570 + 180 H -0.39116243200286149 + 181 H -0.38166561972706725 + 182 O 0.64780287840810935 + 183 H -0.38260631132431167 + 184 H -0.38476220626925972 + 185 O 0.62211481914551481 + 186 H -0.38381531485769760 + 187 H -0.38799420545609231 + 188 O 0.66171710360223379 + 189 H -0.38640461442453788 + 190 H -0.38215089085834897 + 191 O 0.65257516641836677 + 192 H -0.38317921215679529 + 193 H -0.38798578773062242 + 194 O 0.68209216708275378 + 195 H -0.39714375933507828 + 196 H -0.38547191962063021 + 197 O 0.64336181483855359 + 198 H -0.38651757668281683 + 199 H -0.38276486846342583 + 200 O 0.64958033572627194 + 201 H -0.38085267905139586 + 202 H -0.38338731812501481 + 203 O 0.61119591206047108 + 204 H -0.38306285943501095 + 205 H -0.38299499502653556 + 206 O 0.61515044931978125 + 207 H -0.38356168980813699 + 208 H -0.38388172556888955 + 209 O 0.64211704076982201 + 210 H -0.38408151631059828 + 211 H -0.38500225471000726 + 212 O 0.64873254900008526 + 213 H -0.38290771103533794 + 214 H -0.38757256078164437 + 215 O 0.65786661767194854 + 216 H -0.38758208258894811 + 217 H -0.38833119113424874 + 218 O 0.60323454233107032 + 219 H -0.38236661602340249 + 220 H -0.38433613423468960 + 221 O 0.62409552606179552 + 222 H -0.38737576456748191 + 223 H -0.38293757247369964 + 224 O 0.60869696050420607 + 225 H -0.38326636945256198 + 226 H -0.38564133238539988 + 227 O 0.66763155712650857 + 228 H -0.38687397090242870 + 229 H -0.38646215478652435 + 230 O 0.65757958201066646 + 231 H -0.38385699218984004 + 232 H -0.38190000073041175 + 233 O 0.63852217728445559 + 234 H -0.38513060512857245 + 235 H -0.38428347698072030 + 236 O 0.61350492733636663 + 237 H -0.38468182639188897 + 238 H -0.38539078054099118 + 239 O 0.65334237266432993 + 240 H -0.38581660018478026 + 241 H -0.38529378655888213 + 242 O 0.65313419778199489 + 243 H -0.38520359113499614 + 244 H -0.38398297014967386 + 245 O 0.62491894103860712 + 246 H -0.38491811026513045 + 247 H -0.38439931796994042 + 248 O 0.66442688004916750 + 249 H -0.38430862324754278 + 250 H -0.38348371358233035 + 251 O 0.64037511754996856 + 252 H -0.38474205159406361 + 253 H -0.38266857559150347 + 254 O 0.63505651351883152 + 255 H -0.38467157908611060 + 256 H -0.38676877538739041 + 257 O 0.64637597010395798 + 258 H -0.38609162828711374 + 259 H -0.38388924763936738 + 260 O 0.65844589631668704 + 261 H -0.38281724522328220 + 262 H -0.38481306309278263 + 263 O 0.65662742792211315 + 264 H -0.38469109720402017 + 265 H -0.38448488320095653 + 266 O 0.65909431812519870 + 267 H -0.38563528522701196 + 268 H -0.38493270943074420 + 269 O 0.63772861798929981 + 270 H -0.38399947001702261 + 271 H -0.38360906525456329 + 272 O 0.64715559191300542 + 273 H -0.38481490025647014 + 274 H -0.38383142679871352 + 275 O 0.65983854464200409 + 276 H -0.38603396323834538 + 277 H -0.38308994208739355 + 278 O 0.63603608218827556 + 279 H -0.38692798845397547 + 280 H -0.38864604877136599 + 281 O 0.62828867865037008 + 282 H -0.39580455241394807 + 283 H -0.38853989357943242 + 284 O 0.66894011974030043 + 285 H -0.38835593846051564 + 286 H -0.38412741456803612 + 287 O 0.65787668243718400 + 288 H -0.38435435026896658 + 289 H -0.39067660690435058 + 290 O 0.65260435683509499 + 291 H -0.38921350126075760 + 292 H -0.38337851791964173 + 293 O 0.64876419705635957 + 294 H -0.38255343486336202 + 295 H -0.38456644513480631 + 296 O 0.63383606799740377 + 297 H -0.38380323087890689 + 298 H -0.38730361473968378 + 299 O 0.70697947795857807 + 300 H -0.39086661652135513 + Used mem After gpmd_FirstCharges (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + Used mem Before gpmd_DM_Min (pid 25769, ppid 25763) = 1905 MiB (2244 MiB) + SCF iter 1 + + At routine gpmdcov_DM_Min; rank 1 SCF iter 1 + + + At routine gpmdcov_DM_Min; In real Coul ... + + Time for Real coul = 13 ms + + At routine gpmdcov_DM_Min; In recip Coul ... + + Time for Recip coul = 12 ms + + At routine gpmdcov_DM_Min; In prg_get_hscf ... + + starting solver ... + In get_density_t_efd ... + ipt = 1 Ef = 9.8908488652095117E-002 + Time for Solver = 699 ms +orthop_bml + leaving solver ... + + At routine gpmdcov_DM_Min; Entering prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Leaving prg_deorthogonalize... + + + At routine gpmdcov_DM_Min; Getting system charges... diff --git a/examples/gpmdk/tests/run_test.sh b/examples/gpmdk/tests/run_test.sh new file mode 100755 index 00000000..e74eb2ad --- /dev/null +++ b/examples/gpmdk/tests/run_test.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +export OMP_NUM_THREADS=20 +source ../vars +RUN="$HOME/gpmd/build/gpmdcov" +#RUN="/home/cnegre/qmd-progress/build/gpmdcov" +#rm *.tmp + +declare -A Ref +Ref["KerFullBuildAlways"]=16 +Ref["KerFullBuildOnce"]=17 +Ref["KerFullBuildOnceBlock"]=20 +Ref["KerFullBuildAlwaysBlock"]=20 +Ref["KerFullBuildOnceMetis"]=23 +Ref["KerFullBuildAlwaysMetis"]=23 +Ref["KerByPartsBuildOnceBlockUpdate"]=21 +Ref["KerByPartsBuildOnceMetisUpdate"]=20 +Ref["KerByPartsBuildOnceBlock"]=18 +Ref["KerByPartsBuildAlwaysMetis"]=18 + +if [ 1 == 1 ] +then +for name in KerFullBuildAlways KerFullBuildOnce KerFullBuildOnceBlock \ + KerFullBuildAlwaysBlock KerFullBuildOnceMetis KerFullBuildAlwaysMetis \ + KerByPartsBuildOnceBlockUpdate KerByPartsBuildOnceMetisUpdate ; do + + cp ./input_$name'.in' input.tmp + if [ "$1" = "mpi" ] + then + echo "Testing with MPI ..." + #mpirun -np 2 $RUN input.tmp &> out.tmp + mpirun -np 2 ../build/gpmdcov input.tmp &> out.tmp + else + $RUN input.tmp &> out.tmp + fi + nscf=`grep "SCF converged" out.tmp | awk 'NF>1{print $(NF-2)}'` + echo $nscf + if [ "$nscf" -le "${Ref[$name]}" ] + then + echo "Test for "$name" ... OK" + else + echo "ERROR at "$name + exit -1 + fi + +done +fi + + +for name in DiagEf BlockPart KerUpdate; do + cp ./input_$name'.in' input.tmp + if [ "$1" = "mpi" ] + then + mpirun -np 2 $RUN input.tmp &> out.tmp + else + $RUN input.tmp &> out.tmp + fi + ./get_energy.py out.tmp &> energy.tmp + result=`./test-energy.py --reference ./ref.$name --current energy.tmp ` + if [ "$result" != "Error" ] + then + echo "ERROR at "$name + exit -1 + else + echo "Test for "$name" ... OK" + fi +done + diff --git a/examples/gpmdk/tests/test-energy.py b/examples/gpmdk/tests/test-energy.py new file mode 100755 index 00000000..8454d54d --- /dev/null +++ b/examples/gpmdk/tests/test-energy.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +def compare_MD(reference, current, reltol): + """Compare MD energies. + + Given a reference output and the current output, compare the MD + energies to within the relative tolerance given by reltol. + + """ + + import sys + + fd = open(reference) + reference_energies = [] + for line in fd: + result = line.split() + reference_energies.append(float(result[0])) + fd.close() + + fd = open(current) + current_energies = [] + for line in fd: + result = line.split() + current_energies.append(float(result[0])) + fd.close() + + for i in range(len(reference_energies)): + diff = abs(reference_energies[i] - current_energies[i]) + if reference_energies[i] != 0: + diff = abs(diff/reference_energies[i]) + if diff > reltol: + print("Error") + exit(0) + print("Ok") + +def main(): + """The main function. + """ + + import argparse, os, sys + + parser = argparse.ArgumentParser(description="""Script to compare MD results by using the total energy""") + parser.add_argument("--reference", + help="The reference output") + parser.add_argument("--current", + help="The current output") + parser.add_argument("--reltol", + help="Relative tolerance when comparing, default is %(default)s", + type=float, + default=1e-10) + options = parser.parse_args() + + compare_MD(options.reference, options.current, options.reltol) + +if __name__ == "__main__": + main() From f2a3c0851ad69f92c6a3118f0a7292a7ac101060 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 2 Jul 2024 09:39:38 -0600 Subject: [PATCH 02/69] Add build script for hackathon --- build_chicoma_hackathon.sh | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 build_chicoma_hackathon.sh diff --git a/build_chicoma_hackathon.sh b/build_chicoma_hackathon.sh new file mode 100755 index 00000000..56b94bd5 --- /dev/null +++ b/build_chicoma_hackathon.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Make a copy of this script and modify the METIS and BML library locations + +rm -r build +#rm -r install_magma_2.7.2 +rm -r install_hackathon + +# Set METIS and BML Library locations +METIS_LIB="/usr/projects/icapt/mewall/packages/gpmd/metis-5.1.0/" +export BML_LIB=${BML_LIB:="/usr/projects/icapt/mewall/packages/gpmd/qmd-progress/bml/install_hackathon"} +#BML_LIB="/usr/projects/icapt/mewall/packages/gpmd/bml/install_magma_2.7.2" + +MY_PATH=`pwd` + +# Configuring PROGRESS with OpenMP +export CC=${CC:=cc} +export FC=${FC:=ftn} +export CXX=${CXX:=CC} +export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ffixed-line-length-none"} +#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"} +export PKG_CONFIG_PATH="$BML_LIB/lib/pkgconfig:$BML_LIB/lib64/pkgconfig:$PKG_CONFIG_PATH" +export PROGRESS_OPENMP=${PROGRESS_OPENMP:=yes} +export PROGRESS_MPI=${PROGRESS_MPI:=yes} +export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_hackathon"} +export PROGRESS_GRAPHLIB=${PROGRESS_GRAPHLIB:=yes} +export PROGRESS_TESTING=${PROGRESS_TESTING:=yes} +export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release} +export PROGRESS_EXAMPLES=${PROGRESS_EXAMPLES:=yes} +#export PROGRESS_BENCHMARKS=yes +export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=$METIS_LIB/} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch -fdefault-integer-8"} +export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +export CMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH:="$METIS_LIB/include"} +export CMAKE_LIBRARY_PATH=${CMAKE_LIBRARY_PATH:="$METIS_LIB/lib"} +#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" +./build.sh configure + +# Make PROGRESS library and examples after running this script: +# cd build +# make From 3efdf3785fe2f72c3bc31f1946428e5dd2e46312 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 2 Jul 2024 14:54:14 -0600 Subject: [PATCH 03/69] Update bml submodule --- bml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bml b/bml index 0e94c8e2..5ad7c265 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 0e94c8e25892bbd48b94603a9a204d3d73eec85e +Subproject commit 5ad7c26563b74d5c236ba67714c667aa8766840d From 8c1bcab44a0d470f78956efa32a3a7bc7f57df76 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 3 Jul 2024 08:58:12 -0600 Subject: [PATCH 04/69] Update bml submodule --- bml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bml b/bml index 5ad7c265..25bf4f6f 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 5ad7c26563b74d5c236ba67714c667aa8766840d +Subproject commit 25bf4f6fbfdcd2ff4266c233c7dd1d9e77b4be1f From 48b80223d8a419d1fb00ec8c5c760bb0cdc3fe85 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 3 Jul 2024 12:24:19 -0600 Subject: [PATCH 05/69] Add electrons.dat to latteTBparams --- examples/gpmdk/tests/latteTBparams/electrons.dat | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 examples/gpmdk/tests/latteTBparams/electrons.dat diff --git a/examples/gpmdk/tests/latteTBparams/electrons.dat b/examples/gpmdk/tests/latteTBparams/electrons.dat new file mode 100755 index 00000000..489deede --- /dev/null +++ b/examples/gpmdk/tests/latteTBparams/electrons.dat @@ -0,0 +1,8 @@ +Noelem= 5 +Element basis Numel Es Ep Ed Ef Mass HubbardU Wss Wpp Wdd Wff +N sp 5.000000 -18.556500 -7.062500 0.000000 0.000000 14.00307369 17.372900 0.000000 -0.693400 0.000000 0.000000 +O sp 6.000000 -23.937700 -9.003500 0.000000 0.000000 15.9949150 11.876141 0.000000 -0.7576500 0.000000 0.000000 +H s 1.000000 -6.483500 0.000000 0.000000 0.000000 1.007825016 12.054683 -2.23400 0.000000 0.000000 0.000000 +C sp 4.000000 -13.719900 -5.254100 0.000000 0.000000 12.000000 14.240811 0.000000 -0.6181000 0.000000 0.000000 +W sd 6.0 -4.05 0.0 -2.12 0.0 183.84 7.048 0.0 0.0 0.0 0.0 + From 05314a5cc02f4fecd7c874a500f9de35c5a1eaeb Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 5 Jul 2024 11:43:35 -0600 Subject: [PATCH 06/69] Add MPI barrier to pin down performance issue --- examples/gpmdk/run/coords_2088.dat | 2092 --------------------------- examples/gpmdk/run/coords_300.dat | 304 ---- examples/gpmdk/run/get_energy.py | 17 - examples/gpmdk/run/gpmd_input.in | 153 -- examples/gpmdk/run/input.in | 182 --- examples/gpmdk/run/job | 19 - examples/gpmdk/run/latte.in | 74 - examples/gpmdk/run/run.sh | 14 - examples/gpmdk/run/sbatch.sh | 20 - examples/gpmdk/run/wrapper.sh | 10 - examples/gpmdk/src/gpmdcov_part.F90 | 12 +- 11 files changed, 7 insertions(+), 2890 deletions(-) delete mode 100644 examples/gpmdk/run/coords_2088.dat delete mode 100644 examples/gpmdk/run/coords_300.dat delete mode 100755 examples/gpmdk/run/get_energy.py delete mode 100644 examples/gpmdk/run/gpmd_input.in delete mode 100644 examples/gpmdk/run/input.in delete mode 100755 examples/gpmdk/run/job delete mode 100644 examples/gpmdk/run/latte.in delete mode 100755 examples/gpmdk/run/run.sh delete mode 100755 examples/gpmdk/run/sbatch.sh delete mode 100644 examples/gpmdk/run/wrapper.sh diff --git a/examples/gpmdk/run/coords_2088.dat b/examples/gpmdk/run/coords_2088.dat deleted file mode 100644 index e0a6c820..00000000 --- a/examples/gpmdk/run/coords_2088.dat +++ /dev/null @@ -1,2092 +0,0 @@ - 2088 - 27.83000 0.00000 0.00000 - 0.00000 27.83000 0.00000 - 0.00000 0.00000 27.83000 - O 13.72700 13.73400 13.91500 - H 13.30900 14.59500 13.91500 - H 14.70800 13.41500 13.91500 -O 2.30000 6.28000 1.13000 -H 1.37000 6.26000 1.50000 -H 2.31000 5.89000 0.21000 -O 5.69000 12.75100 11.65100 -H 4.76000 12.68100 11.28100 -H 5.80000 13.64100 12.09100 -O 15.55100 15.11100 7.03000 -H 14.98100 14.95100 7.84000 -H 14.96100 15.21100 6.23000 -O 17.43100 6.18000 8.56000 -H 17.76100 7.12000 8.56000 -H 17.94100 5.64000 9.22000 -O 11.35100 7.03000 7.17000 -H 11.92100 7.81000 6.92000 -H 10.75100 7.29000 7.93000 -O 17.55100 6.07000 2.31000 -H 17.43100 5.94000 1.32000 -H 17.25100 5.26000 2.80000 -O 7.68000 11.44100 10.23100 -H 6.90000 11.61100 10.83100 -H 8.02000 12.31100 9.87100 -O 8.50000 7.98000 18.23100 -H 8.46000 8.74000 18.88100 -H 8.72000 8.34000 17.32100 -O 6.85000 10.12100 6.65000 -H 7.54000 9.96100 7.35000 -H 6.12000 10.69100 7.03000 -O 6.86000 11.61100 18.03100 -H 7.46000 12.40100 18.17100 -H 6.00000 11.92100 17.62100 -O 3.35000 14.35100 10.61100 -H 2.57000 14.04100 10.08100 -H 3.93000 14.93100 10.04100 -O 4.38000 3.92000 14.99100 -H 5.20000 3.36000 15.08100 -H 3.57000 3.34000 15.03100 -O 16.03100 4.47000 7.37000 -H 15.29100 4.93000 6.87000 -H 16.54100 5.15000 7.90000 -O 2.31000 17.13100 4.83000 -H 2.65000 17.90100 5.37000 -H 2.75000 17.13100 3.93000 -O 11.27100 13.41100 16.90100 -H 11.74100 13.41100 17.78100 -H 10.79100 12.54100 16.79100 -O 2.40000 10.91100 8.86000 -H 2.54000 10.07100 9.38000 -H 1.85000 11.55100 9.41000 -O 6.20000 17.86100 14.39100 -H 5.28000 17.69100 14.74100 -H 6.48000 18.78100 14.65100 -O 6.06000 9.64100 1.23000 -H 6.13000 10.48100 0.69000 -H 6.52000 9.77100 2.11000 -O 15.94100 1.14000 14.80100 -H 15.76100 1.81000 14.08100 -H 15.91100 1.60000 15.69100 -O 18.42100 17.67100 3.59000 -H 18.96100 17.38100 4.39000 -H 18.72100 18.57100 3.30000 -O 16.89100 9.22000 6.12000 -H 17.84100 8.93000 6.20000 -H 16.81100 9.87000 5.37000 -O 16.41100 11.08100 4.32000 -H 17.27100 11.10100 3.80000 -H 16.55100 11.55100 5.20000 -O 6.13000 13.65100 7.26000 -H 5.64000 12.78100 7.35000 -H 5.90000 14.08100 6.39000 -O 8.09000 0.04000 5.02000 -H 8.49000 0.95000 4.93000 -H 7.09000 0.12000 5.08000 -O 15.25100 9.99100 1.90000 -H 14.62100 9.23100 2.03000 -H 15.73100 10.17100 2.76000 -O 11.87100 17.92100 16.16100 -H 12.11100 18.52100 15.40100 -H 11.94100 16.97100 15.86100 -O 3.17000 2.51000 18.01100 -H 3.88000 3.22000 18.07100 -H 2.29000 2.90000 18.29100 -O 14.66100 14.17100 9.53100 -H 14.07100 14.23100 10.33100 -H 14.51100 13.29100 9.07100 -O 16.67100 17.14100 5.72000 -H 16.26100 16.91100 4.84000 -H 16.49100 16.40100 6.37000 -O 5.98000 7.29000 2.70000 -H 6.22000 7.98000 2.02000 -H 5.20000 7.62000 3.24000 -O 12.81100 3.45000 9.44100 -H 11.95100 2.95000 9.31100 -H 13.43100 2.91000 10.00100 -O 15.76100 16.62100 3.07000 -H 16.65100 17.08100 3.10000 -H 15.55100 16.38100 2.12000 -O 8.07000 6.05000 14.65100 -H 7.60000 6.02000 15.54100 -H 7.56000 5.50000 13.99100 -O 13.94100 4.69000 16.74100 -H 13.74100 5.12000 17.62100 -H 14.72100 4.07000 16.83100 -O 9.73100 8.90000 15.72100 -H 10.19100 8.06000 15.43100 -H 9.17100 9.24000 14.97100 -O 9.91100 4.10000 12.42100 -H 9.14100 4.44000 12.96100 -H 9.57100 3.59000 11.63100 -O 10.41100 7.01000 4.29000 -H 10.67100 6.97000 5.25000 -H 9.56100 6.50000 4.15000 -O 8.65000 3.48000 1.95000 -H 9.24000 4.11000 1.46000 -H 8.84000 2.54000 1.66000 -O 17.19100 5.85000 18.31100 -H 16.93100 6.74000 17.95100 -H 17.17100 5.17000 17.58100 -O 13.62100 11.44100 5.45000 -H 14.45100 11.15100 4.97000 -H 13.13100 12.11100 4.89000 -O 5.50000 1.96000 8.85000 -H 5.45000 1.91000 9.85000 -H 5.52000 2.92000 8.56000 -O 10.08100 14.56100 4.77000 -H 9.62100 15.28100 4.25000 -H 10.04100 14.76100 5.75000 -O 3.51000 18.01100 8.53000 -H 4.01000 17.15100 8.59000 -H 4.16000 18.78100 8.50000 -O 17.95100 10.66100 8.73000 -H 17.33100 10.51100 7.97000 -H 17.43100 10.77100 9.58000 -O 12.27100 15.50100 15.06100 -H 12.33100 14.73100 15.70100 -H 11.75100 15.24100 14.26100 -O 3.21000 9.43100 2.42000 -H 4.03000 9.82100 2.00000 -H 2.94000 8.61100 1.93000 -O 14.58100 7.35000 7.28000 -H 14.53100 6.70000 8.03000 -H 15.38100 7.94000 7.41000 -O 4.61000 12.66100 17.27100 -H 4.11000 12.67100 16.41100 -H 3.98000 12.48100 18.03100 -O 11.11100 17.76100 2.37000 -H 10.51100 17.14100 2.87000 -H 11.42100 17.32100 1.52000 -O 2.02000 2.85000 14.98100 -H 1.22000 3.45000 14.85100 -H 1.92000 2.36000 15.84100 -O 16.32100 13.77100 0.81000 -H 16.00100 14.71100 0.71000 -H 15.56100 13.14100 0.69000 -O 4.64000 17.43100 3.23000 -H 4.97000 17.82100 4.09000 -H 5.40000 17.36100 2.58000 -O 14.00100 1.07000 4.26000 -H 13.76100 0.70000 3.36000 -H 14.99100 1.23000 4.30000 -O 9.40100 16.98100 9.04000 -H 10.20100 16.41100 9.25000 -H 8.91100 16.58100 8.27000 -O 3.82000 7.00000 4.80000 -H 4.27000 6.10000 4.77000 -H 2.88000 6.89000 5.13000 -O 15.47100 2.22000 17.29100 -H 15.42100 2.59000 18.21100 -H 14.75100 1.53000 17.17100 -O 6.14000 1.22000 1.17000 -H 7.12000 1.00000 1.24000 -H 5.83000 1.05000 0.24000 -O 7.81000 2.64000 17.49100 -H 8.48000 2.03000 17.92100 -H 7.08000 2.83000 18.14100 -O 8.88000 15.14100 11.95100 -H 8.65000 14.89100 11.01100 -H 9.49000 14.45100 12.34100 -O 13.51100 5.90000 14.33100 -H 13.79100 5.47000 15.18100 -H 13.76100 6.86000 14.34100 -O 8.03000 14.02100 9.24000 -H 8.93000 14.16100 8.82000 -H 7.32000 14.04100 8.53000 -O 9.22000 5.03000 8.99000 -H 8.97000 4.94000 8.03000 -H 9.70000 4.21000 9.30000 -O 5.39000 0.64000 5.12000 -H 4.58000 0.65000 5.70000 -H 5.42000 1.47000 4.57000 -O 14.34100 11.88100 0.41000 -H 14.66100 11.12100 0.98000 -H 13.42100 12.15100 0.71000 -O 2.97000 0.35000 1.71000 -H 3.46000 1.19000 1.50000 -H 3.59000 -0.30000 2.16000 -O 9.35100 2.36000 4.80000 -H 8.87100 2.77000 4.02000 -H 10.34100 2.34000 4.61000 -O 10.76100 6.83000 14.64100 -H 9.96100 6.22000 14.67100 -H 11.57100 6.30000 14.40100 -O 12.27100 15.70100 7.93000 -H 12.48100 16.44100 7.28000 -H 12.95100 15.70100 8.66000 -O 4.59000 11.52100 7.41000 -H 3.88000 11.25100 8.06000 -H 4.33000 11.24100 6.48000 -O 12.71100 17.97100 5.91000 -H 13.15100 18.61100 5.27000 -H 12.21100 18.49100 6.61000 -O 10.32100 5.49000 0.16000 -H 9.91100 6.31000 -0.23000 -H 10.96100 5.75000 0.89000 -O 0.78000 5.56000 13.86100 -H 1.70000 5.55000 13.45100 -H 0.72000 6.30000 14.53100 -O 5.61000 2.22000 11.47100 -H 5.99000 1.38000 11.84100 -H 4.73000 2.41000 11.91100 -O 8.66000 4.54000 6.42000 -H 8.34000 5.26000 5.80000 -H 8.90000 3.73000 5.89000 -O 10.17100 0.39000 7.53000 -H 9.45100 0.44000 6.84000 -H 9.93100 -0.30000 8.22000 -O 14.29100 11.73100 8.67000 -H 13.74100 10.89100 8.60000 -H 14.55100 12.02100 7.75000 -O 14.66100 5.90000 9.92100 -H 14.36100 4.95000 9.99100 -H 15.39100 6.06000 10.58100 -O 18.57100 8.33000 3.77000 -H 18.99100 7.69000 4.41000 -H 18.19100 7.82000 2.99000 -O 4.88000 13.85100 1.74000 -H 4.01000 13.70100 2.21000 -H 4.71000 14.11100 0.79000 -O 16.64100 12.80100 6.57000 -H 17.63100 12.88100 6.71000 -H 16.19100 13.64100 6.88000 -O 13.90100 5.75000 0.78000 -H 13.36100 5.54000 1.59000 -H 14.81100 5.34000 0.87000 -O 5.27000 2.56000 3.28000 -H 5.54000 1.97000 2.53000 -H 5.27000 3.51000 2.97000 -O 17.54100 12.23100 15.88100 -H 18.45100 11.84100 15.75100 -H 17.62100 13.19100 16.12100 -O 10.64100 13.47100 13.40100 -H 9.84100 13.24100 13.95100 -H 11.47100 13.21100 13.89100 -O 15.92100 16.29100 16.25100 -H 16.19100 16.63100 15.35100 -H 16.71100 15.91100 16.71100 -O 11.11100 11.95100 11.00100 -H 10.71100 12.39100 11.81100 -H 10.70100 12.32100 10.17100 -O 16.38100 10.99100 10.79100 -H 16.43100 11.80100 11.38100 -H 15.52100 11.01100 10.28100 -O 9.15000 0.89000 14.02100 -H 9.40000 0.69000 13.07100 -H 9.87000 1.45000 14.44100 -O 9.80100 11.16100 17.19100 -H 8.81100 11.22100 17.29100 -H 10.03100 10.36100 16.63100 -O 7.05000 10.50100 3.68000 -H 6.91000 10.57100 4.67000 -H 7.89000 9.99100 3.50000 -O 4.10000 8.13000 12.51100 -H 4.96000 8.25000 13.01100 -H 3.68000 7.26000 12.78100 -O 12.74100 3.86000 12.62100 -H 12.95100 4.60000 13.26100 -H 11.85100 4.03000 12.19100 -O 3.67000 11.00100 5.01000 -H 3.60000 11.83100 4.45000 -H 3.71000 10.20100 4.41000 -O 5.66000 5.37000 8.65000 -H 5.78000 6.03000 7.91000 -H 6.12000 5.71000 9.48000 -O 12.52100 13.48100 3.88000 -H 13.02100 14.25100 4.28000 -H 11.57100 13.50100 4.20000 -O 12.72100 14.45100 11.42100 -H 13.19100 14.58100 12.29100 -H 12.06100 13.71100 11.51100 -O 15.82100 6.39000 4.72000 -H 15.51100 7.00000 5.45000 -H 16.32100 6.91000 4.03000 -O 3.54000 15.10100 13.29100 -H 3.33000 14.66100 12.42100 -H 4.51000 15.36100 13.32100 -O 4.02000 7.51000 15.98100 -H 4.70000 8.06000 15.51100 -H 4.42000 6.63000 16.25100 -O 15.87100 7.79000 16.70100 -H 14.95100 8.17000 16.65100 -H 16.47100 8.26000 16.05100 -O 10.13100 1.05000 17.70100 -H 10.19100 1.90000 17.18100 -H 10.45100 0.29000 17.13100 -O 5.04000 0.50000 17.40100 -H 4.62000 -0.07000 16.70100 -H 4.38000 1.19000 17.72100 -O 5.73000 8.70000 10.29100 -H 6.17000 9.59000 10.20100 -H 5.10000 8.70000 11.06100 -O 13.60100 8.62000 10.45100 -H 12.85100 8.62000 9.79100 -H 13.97100 7.70000 10.54100 -O 12.09100 5.25000 2.75000 -H 12.22100 4.41000 3.29000 -H 11.80100 5.99000 3.35000 -O 3.07000 2.13000 12.31100 -H 2.84000 2.50000 13.21100 -H 2.77000 1.18000 12.25100 -O 0.37000 13.10100 12.82100 -H 0.90000 12.61100 13.50100 -H 0.59000 14.08100 12.87100 -O 7.32000 6.34000 10.64100 -H 7.91000 6.08000 9.88100 -H 7.04000 7.30000 10.53100 -O 17.28100 9.35100 18.54100 -H 16.82100 9.28100 17.65100 -H 16.66100 9.79100 19.20100 -O 16.22100 3.67000 3.74000 -H 16.24100 2.91000 4.38000 -H 15.74100 4.44000 4.14000 -O 10.23100 7.66000 9.66100 -H 10.38100 7.87000 10.62100 -H 9.93100 6.71000 9.57100 -O 9.80100 15.73100 17.00100 -H 9.60100 16.17100 16.12100 -H 10.19100 14.82100 16.84100 -O 18.59100 15.18100 16.05100 -H 18.73100 15.45100 15.10100 -H 19.42100 15.40100 16.58100 -O 3.50000 8.98000 18.04100 -H 4.26000 9.42000 18.52100 -H 3.85000 8.51000 17.22100 -O 15.40100 2.74000 1.25000 -H 14.79100 1.99000 1.48000 -H 15.62100 3.26000 2.08000 -O 13.03100 8.38000 0.42000 -H 13.37100 7.45000 0.57000 -H 13.21100 8.65000 -0.53000 -O 10.68100 13.33100 8.49000 -H 10.75100 12.49100 7.94000 -H 11.30100 14.02100 8.13000 -O 3.19000 8.10000 9.49100 -H 4.12000 8.46000 9.54100 -H 3.13000 7.25000 10.01100 -O 3.39000 5.09000 10.06100 -H 2.87000 4.26000 9.89100 -H 4.16000 5.14000 9.42100 -O 11.38100 3.80000 16.78100 -H 10.93100 4.43000 17.42100 -H 12.31100 4.11000 16.61100 -O 11.60100 2.07000 14.77100 -H 11.60100 2.71000 15.54100 -H 11.88100 2.55000 13.94100 -O 5.04000 15.68100 9.10000 -H 5.70000 16.42100 9.19000 -H 5.48000 14.89100 8.68000 -O 10.02100 7.96000 12.38100 -H 10.43100 7.64000 13.24100 -H 9.06100 7.69000 12.35100 -O 0.40000 5.44000 11.14100 -H 1.25000 5.11000 10.73100 -H 0.53000 5.59000 12.12100 -O 1.89000 5.20000 17.22100 -H 2.48000 4.80000 16.52100 -H 1.31000 5.91000 16.81100 -O 13.69100 9.50100 16.60100 -H 14.08100 10.39100 16.80100 -H 13.79100 9.30100 15.63100 -O 8.15000 5.72000 3.25000 -H 8.22000 4.83000 2.79000 -H 7.21000 6.06000 3.17000 -O 16.57100 6.04000 12.06100 -H 16.19100 5.35000 12.68100 -H 17.39100 5.68000 11.62100 -O 2.52000 15.64100 17.44100 -H 2.22000 16.21100 18.20100 -H 2.45000 14.67100 17.70100 -O 6.71000 4.64000 12.69100 -H 6.37000 3.75000 12.39100 -H 6.97000 5.18000 11.89100 -O 9.30000 16.78100 14.65100 -H 9.06000 16.60100 13.70100 -H 9.60000 17.72100 14.75100 -O 4.73000 5.00000 1.91000 -H 5.34000 5.80000 1.95000 -H 3.78000 5.31000 1.98000 -O 1.59000 11.37100 14.66100 -H 1.81000 10.76100 15.42100 -H 1.69000 10.88100 13.80100 -O 13.47100 10.59100 12.34100 -H 13.71100 9.96100 11.60100 -H 12.57100 10.99100 12.16100 -O 13.02100 8.55000 3.09000 -H 12.16100 8.24000 3.51000 -H 12.98100 8.41000 2.10000 -O 17.59100 17.47100 11.54100 -H 18.20100 17.77100 10.81100 -H 17.21100 16.58100 11.32100 -O 12.52100 17.31100 11.28100 -H 13.36100 17.36100 10.74100 -H 12.29100 16.35100 11.46100 -O 0.83000 12.58100 10.22100 -H 0.78000 12.57100 11.22100 -H 0.00000 12.17100 9.84100 -O 6.88000 16.62100 17.16100 -H 6.32000 17.43100 17.25100 -H 7.40000 16.66100 16.30100 -O 9.03000 0.86000 1.33000 -H 9.54000 0.87000 0.47000 -H 9.59000 0.44000 2.04000 -O 17.26100 1.35000 5.23000 -H 17.99100 1.18000 4.56000 -H 16.95100 0.48000 5.61000 -O 13.88100 15.73100 4.77000 -H 14.55100 15.85100 4.03000 -H 13.48100 16.62100 5.00000 -O 1.30000 17.94100 18.51100 -H 0.89000 17.20100 19.04100 -H 1.94000 18.45100 19.09100 -O 12.80100 9.27000 6.72000 -H 13.40100 8.46000 6.74000 -H 13.20100 9.96000 6.12000 -O 8.30000 12.73100 14.22100 -H 8.25000 13.06100 15.17100 -H 7.44000 12.92100 13.76100 -O 6.72000 16.16100 1.54000 -H 6.81000 16.26100 0.55000 -H 6.32000 15.27100 1.75000 -O 16.50100 17.20100 13.94100 -H 17.03100 17.30100 13.10100 -H 16.23100 18.10100 14.28100 -O 18.41100 1.75000 9.63100 -H 18.80100 0.90000 9.27100 -H 17.43100 1.77000 9.44100 -O 2.63000 3.26000 7.20000 -H 1.84000 3.77000 6.86000 -H 2.54000 3.11000 8.18000 -O 11.94100 16.12100 0.31000 -H 12.00100 15.19100 0.68000 -H 11.35100 16.12100 -0.49000 -O 8.22000 10.02100 13.72100 -H 8.62000 10.01100 12.80100 -H 8.32000 10.94100 14.12100 -O 9.16000 9.10000 2.91000 -H 9.79000 9.48000 2.23000 -H 9.56000 8.27000 3.30000 -O 15.04100 16.07100 0.44000 -H 14.12100 16.44100 0.51000 -H 15.42100 16.27100 -0.46000 -O 3.72000 12.88100 14.90100 -H 3.59000 13.81100 14.56100 -H 2.88000 12.36100 14.77100 -O 10.39100 10.98100 6.96000 -H 9.69100 10.51100 7.50000 -H 10.98100 10.30100 6.53000 -O 10.14100 2.36000 9.71100 -H 10.06100 2.00000 8.78100 -H 10.12100 1.60000 10.36100 -O 5.90000 14.87100 4.91000 -H 6.32000 14.29100 4.21000 -H 5.46000 15.66100 4.47000 -O 17.09100 3.85000 13.81100 -H 17.82100 4.54000 13.85100 -H 17.37100 3.10000 13.22100 -O 2.55000 13.48100 2.90000 -H 1.59000 13.49100 2.63000 -H 2.67000 14.01100 3.74000 -O 6.72000 2.03000 14.89100 -H 7.62000 1.87000 14.49100 -H 6.80000 2.08000 15.88100 -O 14.40100 8.56000 13.98100 -H 13.83100 9.08000 13.35100 -H 15.36100 8.68000 13.74100 -O 0.72000 1.66000 3.18000 -H 0.55000 2.49000 2.64000 -H 1.62000 1.29000 2.96000 -O 11.83100 13.35100 1.19000 -H 10.84100 13.24100 1.21000 -H 12.17100 13.50100 2.12000 -O 6.13000 8.42000 14.31100 -H 6.69000 9.23000 14.14100 -H 6.72000 7.62000 14.34100 -O 14.93100 17.67100 9.59100 -H 15.26100 18.31100 8.90100 -H 15.59100 17.61100 10.34100 -O 7.16000 5.65000 17.08100 -H 7.35000 6.30000 17.82100 -H 7.76000 4.85000 17.17100 -O 14.50100 12.20100 16.33100 -H 14.41100 12.10100 17.32100 -H 15.46100 12.13100 16.07100 -O 3.90000 17.41100 15.60100 -H 2.99000 17.82100 15.58100 -H 3.83000 16.47100 15.92100 -O 16.74100 8.83000 12.54100 -H 16.47100 7.94000 12.17100 -H 16.75100 9.51000 11.81100 -O 12.25100 3.25000 4.49000 -H 12.90100 2.51000 4.38000 -H 12.45100 3.75000 5.33000 -O 5.94000 7.45000 6.52000 -H 6.44000 8.30000 6.33000 -H 5.06000 7.47000 6.04000 -O 17.77100 3.42000 16.42100 -H 17.60100 3.73000 15.48100 -H 16.93100 3.05000 16.80100 -O 17.30100 9.34100 15.17100 -H 17.68100 10.25100 15.32100 -H 17.22100 9.17100 14.18100 -O 8.59000 13.74100 0.16000 -H 8.13000 13.89100 1.04000 -H 9.03000 14.59100 -0.14000 -O 6.61000 17.90100 9.53100 -H 6.15000 18.78100 9.40100 -H 7.60000 18.02100 9.46100 -O 14.08100 18.51100 17.20100 -H 13.12100 18.40100 16.93100 -H 14.64100 17.84100 16.72100 -O 8.59000 9.56100 8.61000 -H 9.13000 8.87100 9.09000 -H 8.27000 10.25100 9.27000 -O 10.83100 9.84100 0.87000 -H 10.60100 10.37100 0.05000 -H 11.64100 9.28100 0.68000 -O 18.61100 15.69100 8.51000 -H 17.90100 15.57100 7.81000 -H 18.62100 14.90100 9.11000 -O 2.21000 13.14100 18.44100 -H 1.56000 12.41100 18.23100 -H 2.25000 13.28100 19.42100 -O 0.79000 12.40100 6.53000 -H 0.78000 11.93100 7.41000 -H 1.61000 12.12100 6.02000 -O 6.72000 13.91100 16.24100 -H 5.94000 13.41100 16.62100 -H 6.69000 14.86100 16.55100 -O 18.24100 1.92000 12.27100 -H 18.20100 1.02000 12.71100 -H 18.27100 1.81000 11.28100 -O 4.28000 4.24000 5.20000 -H 4.58000 3.52000 4.58000 -H 3.89000 3.84000 6.03000 -O 17.05100 14.87100 11.04100 -H 16.12100 14.62100 10.77100 -H 17.31100 14.37100 11.86100 -O 3.17000 5.47000 12.80100 -H 3.55000 4.88000 13.52100 -H 3.57000 5.21000 11.92100 -O 8.12000 15.86100 6.87000 -H 8.44000 15.96100 5.93000 -H 7.33000 15.24100 6.89000 -O 14.24100 2.14000 11.12100 -H 14.76100 1.49000 11.67100 -H 13.75100 2.77000 11.73100 -O 10.01100 0.34000 11.54100 -H 9.38100 -0.38000 11.23100 -H 10.94100 -0.02000 11.54100 -O 7.70000 13.30100 3.01000 -H 7.24000 12.43100 3.18000 -H 8.61000 13.27100 3.42000 -O 6.18000 15.67100 12.84100 -H 6.13000 16.49100 13.41100 -H 7.07000 15.64100 12.39100 -O 13.52100 0.52000 1.68000 -H 13.87100 0.11000 0.84000 -H 12.62100 0.14000 1.88000 -O 13.00100 4.53000 6.91000 -H 12.41100 5.33000 6.95000 -H 13.15100 4.18000 7.84000 -O 15.93100 2.21000 8.82000 -H 15.09100 2.20000 9.36000 -H 15.95100 3.04000 8.26000 -O 0.39000 10.77100 3.00000 -H 1.38000 10.66100 2.91000 -H -0.01000 9.91100 3.32000 -O 8.75000 16.46100 3.37000 -H 7.98000 16.11100 2.83000 -H 8.43000 17.17100 3.99000 -O 2.30000 6.28000 19.75100 -H 1.37000 6.26000 20.12100 -H 2.31000 5.89000 18.83100 -O 15.55100 15.11100 25.65100 -H 14.98100 14.95100 26.46100 -H 14.96100 15.21100 24.85100 -O 17.43100 6.18000 27.18100 -H 17.76100 7.12000 27.18100 -H 17.94100 5.64000 27.84100 -O 11.35100 7.03000 25.79100 -H 11.92100 7.81000 25.54100 -H 10.75100 7.29000 26.55100 -O 17.55100 6.07000 20.93100 -H 17.43100 5.94000 19.94100 -H 17.25100 5.26000 21.42100 -O 7.68000 11.44100 1.02100 -H 6.90000 11.61100 1.62100 -H 8.02000 12.31100 0.66100 -O 6.85000 10.12100 25.27100 -H 7.54000 9.96100 25.97100 -H 6.12000 10.69100 25.65100 -O 2.31000 17.13100 23.45100 -H 2.65000 17.90100 23.99100 -H 2.75000 17.13100 22.55100 -O 2.40000 10.91100 27.48100 -H 2.54000 10.07100 28.00100 -H 1.85000 11.55100 28.03100 -O 6.06000 9.64100 19.85100 -H 6.13000 10.48100 19.31100 -H 6.52000 9.77100 20.73100 -O 18.42100 17.67100 22.21100 -H 18.96100 17.38100 23.01100 -H 18.72100 18.57100 21.92100 -O 16.89100 9.22000 24.74100 -H 17.84100 8.93000 24.82100 -H 16.81100 9.87000 23.99100 -O 16.41100 11.08100 22.94100 -H 17.27100 11.10100 22.42100 -H 16.55100 11.55100 23.82100 -O 6.13000 13.65100 25.88100 -H 5.64000 12.78100 25.97100 -H 5.90000 14.08100 25.01100 -O 8.09000 0.04000 23.64100 -H 8.49000 0.95000 23.55100 -H 7.09000 0.12000 23.70100 -O 15.25100 9.99100 20.52100 -H 14.62100 9.23100 20.65100 -H 15.73100 10.17100 21.38100 -O 16.67100 17.14100 24.34100 -H 16.26100 16.91100 23.46100 -H 16.49100 16.40100 24.99100 -O 5.98000 7.29000 21.32100 -H 6.22000 7.98000 20.64100 -H 5.20000 7.62000 21.86100 -O 12.81100 3.45000 0.23100 -H 11.95100 2.95000 0.10100 -H 13.43100 2.91000 0.79100 -O 15.76100 16.62100 21.69100 -H 16.65100 17.08100 21.72100 -H 15.55100 16.38100 20.74100 -O 10.41100 7.01000 22.91100 -H 10.67100 6.97000 23.87100 -H 9.56100 6.50000 22.77100 -O 8.65000 3.48000 20.57100 -H 9.24000 4.11000 20.08100 -H 8.84000 2.54000 20.28100 -O 13.62100 11.44100 24.07100 -H 14.45100 11.15100 23.59100 -H 13.13100 12.11100 23.51100 -O 10.08100 14.56100 23.39100 -H 9.62100 15.28100 22.87100 -H 10.04100 14.76100 24.37100 -O 3.51000 18.01100 27.15100 -H 4.01000 17.15100 27.21100 -H 4.16000 18.78100 27.12100 -O 17.95100 10.66100 27.35100 -H 17.33100 10.51100 26.59100 -H 17.43100 10.77100 28.20100 -O 3.21000 9.43100 21.04100 -H 4.03000 9.82100 20.62100 -H 2.94000 8.61100 20.55100 -O 14.58100 7.35000 25.90100 -H 14.53100 6.70000 26.65100 -H 15.38100 7.94000 26.03100 -O 11.11100 17.76100 20.99100 -H 10.51100 17.14100 21.49100 -H 11.42100 17.32100 20.14100 -O 16.32100 13.77100 19.43100 -H 16.00100 14.71100 19.33100 -H 15.56100 13.14100 19.31100 -O 4.64000 17.43100 21.85100 -H 4.97000 17.82100 22.71100 -H 5.40000 17.36100 21.20100 -O 14.00100 1.07000 22.88100 -H 13.76100 0.70000 21.98100 -H 14.99100 1.23000 22.92100 -O 3.82000 7.00000 23.42100 -H 4.27000 6.10000 23.39100 -H 2.88000 6.89000 23.75100 -O 6.14000 1.22000 19.79100 -H 7.12000 1.00000 19.86100 -H 5.83000 1.05000 18.86100 -O 5.39000 0.64000 23.74100 -H 4.58000 0.65000 24.32100 -H 5.42000 1.47000 23.19100 -O 14.34100 11.88100 19.03100 -H 14.66100 11.12100 19.60100 -H 13.42100 12.15100 19.33100 -O 9.35100 2.36000 23.42100 -H 8.87100 2.77000 22.64100 -H 10.34100 2.34000 23.23100 -O 4.59000 11.52100 26.03100 -H 3.88000 11.25100 26.68100 -H 4.33000 11.24100 25.10100 -O 12.71100 17.97100 24.53100 -H 13.15100 18.61100 23.89100 -H 12.21100 18.49100 25.23100 -O 10.32100 5.49000 18.78100 -H 9.91100 6.31000 18.39100 -H 10.96100 5.75000 19.51100 -O 8.66000 4.54000 25.04100 -H 8.34000 5.26000 24.42100 -H 8.90000 3.73000 24.51100 -O 10.17100 0.39000 26.15100 -H 9.45100 0.44000 25.46100 -H 9.93100 -0.30000 26.84100 -O 18.57100 8.33000 22.39100 -H 18.99100 7.69000 23.03100 -H 18.19100 7.82000 21.61100 -O 4.88000 13.85100 20.36100 -H 4.01000 13.70100 20.83100 -H 4.71000 14.11100 19.41100 -O 16.64100 12.80100 25.19100 -H 17.63100 12.88100 25.33100 -H 16.19100 13.64100 25.50100 -O 13.90100 5.75000 19.40100 -H 13.36100 5.54000 20.21100 -H 14.81100 5.34000 19.49100 -O 5.27000 2.56000 21.90100 -H 5.54000 1.97000 21.15100 -H 5.27000 3.51000 21.59100 -O 7.05000 10.50100 22.30100 -H 6.91000 10.57100 23.29100 -H 7.89000 9.99100 22.12100 -O 3.67000 11.00100 23.63100 -H 3.60000 11.83100 23.07100 -H 3.71000 10.20100 23.03100 -O 5.66000 5.37000 27.27100 -H 5.78000 6.03000 26.53100 -H 6.12000 5.71000 28.10100 -O 12.52100 13.48100 22.50100 -H 13.02100 14.25100 22.90100 -H 11.57100 13.50100 22.82100 -O 15.82100 6.39000 23.34100 -H 15.51100 7.00000 24.07100 -H 16.32100 6.91000 22.65100 -O 12.09100 5.25000 21.37100 -H 12.22100 4.41000 21.91100 -H 11.80100 5.99000 21.97100 -O 16.22100 3.67000 22.36100 -H 16.24100 2.91000 23.00100 -H 15.74100 4.44000 22.76100 -O 15.40100 2.74000 19.87100 -H 14.79100 1.99000 20.10100 -H 15.62100 3.26000 20.70100 -O 13.03100 8.38000 19.04100 -H 13.37100 7.45000 19.19100 -H 13.21100 8.65000 18.09100 -O 10.68100 13.33100 27.11100 -H 10.75100 12.49100 26.56100 -H 11.30100 14.02100 26.75100 -O 8.15000 5.72000 21.87100 -H 8.22000 4.83000 21.41100 -H 7.21000 6.06000 21.79100 -O 4.73000 5.00000 20.53100 -H 5.34000 5.80000 20.57100 -H 3.78000 5.31000 20.60100 -O 13.02100 8.55000 21.71100 -H 12.16100 8.24000 22.13100 -H 12.98100 8.41000 20.72100 -O 17.26100 1.35000 23.85100 -H 17.99100 1.18000 23.18100 -H 16.95100 0.48000 24.23100 -O 13.88100 15.73100 23.39100 -H 14.55100 15.85100 22.65100 -H 13.48100 16.62100 23.62100 -O 12.80100 9.27000 25.34100 -H 13.40100 8.46000 25.36100 -H 13.20100 9.96000 24.74100 -O 6.72000 16.16100 20.16100 -H 6.81000 16.26100 19.17100 -H 6.32000 15.27100 20.37100 -O 18.41100 1.75000 0.42100 -H 18.80100 0.90000 0.06100 -H 17.43100 1.77000 0.23100 -O 2.63000 3.26000 25.82100 -H 1.84000 3.77000 25.48100 -H 2.54000 3.11000 26.80100 -O 11.94100 16.12100 18.93100 -H 12.00100 15.19100 19.30100 -H 11.35100 16.12100 18.13100 -O 9.16000 9.10000 21.53100 -H 9.79000 9.48000 20.85100 -H 9.56000 8.27000 21.92100 -O 15.04100 16.07100 19.06100 -H 14.12100 16.44100 19.13100 -H 15.42100 16.27100 18.16100 -O 10.39100 10.98100 25.58100 -H 9.69100 10.51100 26.12100 -H 10.98100 10.30100 25.15100 -O 5.90000 14.87100 23.53100 -H 6.32000 14.29100 22.83100 -H 5.46000 15.66100 23.09100 -O 2.55000 13.48100 21.52100 -H 1.59000 13.49100 21.25100 -H 2.67000 14.01100 22.36100 -O 0.75000 3.45000 18.95100 -H -0.17000 3.17000 18.66100 -H 1.06000 4.22000 18.39100 -O 0.72000 1.66000 21.80100 -H 0.55000 2.49000 21.26100 -H 1.62000 1.29000 21.58100 -O 11.83100 13.35100 19.81100 -H 10.84100 13.24100 19.83100 -H 12.17100 13.50100 20.74100 -O 12.25100 3.25000 23.11100 -H 12.90100 2.51000 23.00100 -H 12.45100 3.75000 23.95100 -O 5.94000 7.45000 25.14100 -H 6.44000 8.30000 24.95100 -H 5.06000 7.47000 24.66100 -O 8.59000 13.74100 18.78100 -H 8.13000 13.89100 19.66100 -H 9.03000 14.59100 18.48100 -O 6.61000 17.90100 0.32100 -H 6.15000 18.78100 0.19100 -H 7.60000 18.02100 0.25100 -O 8.59000 9.56100 27.23100 -H 9.13000 8.87100 27.71100 -H 8.27000 10.25100 27.89100 -O 10.83100 9.84100 19.49100 -H 10.60100 10.37100 18.67100 -H 11.64100 9.28100 19.30100 -O 18.61100 15.69100 27.13100 -H 17.90100 15.57100 26.43100 -H 18.62100 14.90100 27.73100 -O 0.79000 12.40100 25.15100 -H 0.78000 11.93100 26.03100 -H 1.61000 12.12100 24.64100 -O 4.28000 4.24000 23.82100 -H 4.58000 3.52000 23.20100 -H 3.89000 3.84000 24.65100 -O 8.12000 15.86100 25.49100 -H 8.44000 15.96100 24.55100 -H 7.33000 15.24100 25.51100 -O 7.70000 13.30100 21.63100 -H 7.24000 12.43100 21.80100 -H 8.61000 13.27100 22.04100 -O 13.52100 0.52000 20.30100 -H 13.87100 0.11000 19.46100 -H 12.62100 0.14000 20.50100 -O 13.00100 4.53000 25.53100 -H 12.41100 5.33000 25.57100 -H 13.15100 4.18000 26.46100 -O 0.39000 10.77100 21.62100 -H 1.38000 10.66100 21.53100 -H -0.01000 9.91100 21.94100 -O 8.75000 16.46100 21.99100 -H 7.98000 16.11100 21.45100 -H 8.43000 17.17100 22.61100 -O 2.30000 24.90100 1.13000 -H 1.37000 24.88100 1.50000 -H 2.31000 24.51100 0.21000 -O 17.43100 24.80100 8.56000 -H 17.76100 25.74100 8.56000 -H 17.94100 24.26100 9.22000 -O 11.35100 25.65100 7.17000 -H 11.92100 26.43100 6.92000 -H 10.75100 25.91100 7.93000 -O 17.55100 24.69100 2.31000 -H 17.43100 24.56100 1.32000 -H 17.25100 23.88100 2.80000 -O 8.50000 26.60100 18.23100 -H 8.46000 27.36100 18.88100 -H 8.72000 26.96100 17.32100 -O 6.85000 0.91100 6.65000 -H 7.54000 0.75100 7.35000 -H 6.12000 1.48100 7.03000 -O 4.38000 22.54100 14.99100 -H 5.20000 21.98100 15.08100 -H 3.57000 21.96100 15.03100 -O 15.94100 19.76100 14.80100 -H 15.76100 20.43100 14.08100 -H 15.91100 20.22100 15.69100 -O 1.13000 25.99100 15.97100 -H 2.01000 25.86100 16.42100 -H 1.00000 26.96100 15.75100 -O 8.09000 18.66100 5.02000 -H 8.49000 19.57100 4.93000 -H 7.09000 18.74100 5.08000 -O 3.17000 21.13100 18.01100 -H 3.88000 21.84100 18.07100 -H 2.29000 21.52100 18.29100 -O 5.98000 25.91100 2.70000 -H 6.22000 26.60100 2.02000 -H 5.20000 26.24100 3.24000 -O 12.81100 22.07100 9.44100 -H 11.95100 21.57100 9.31100 -H 13.43100 21.53100 10.00100 -O 8.07000 24.67100 14.65100 -H 7.60000 24.64100 15.54100 -H 7.56000 24.12100 13.99100 -O 13.94100 23.31100 16.74100 -H 13.74100 23.74100 17.62100 -H 14.72100 22.69100 16.83100 -O 9.91100 22.72100 12.42100 -H 9.14100 23.06100 12.96100 -H 9.57100 22.21100 11.63100 -O 10.41100 25.63100 4.29000 -H 10.67100 25.59100 5.25000 -H 9.56100 25.12100 4.15000 -O 8.65000 22.10100 1.95000 -H 9.24000 22.73100 1.46000 -H 8.84000 21.16100 1.66000 -O 17.19100 24.47100 18.31100 -H 16.93100 25.36100 17.95100 -H 17.17100 23.79100 17.58100 -O 5.50000 20.58100 8.85000 -H 5.45000 20.53100 9.85000 -H 5.52000 21.54100 8.56000 -O 14.58100 25.97100 7.28000 -H 14.53100 25.32100 8.03000 -H 15.38100 26.56100 7.41000 -O 2.02000 21.47100 14.98100 -H 1.22000 22.07100 14.85100 -H 1.92000 20.98100 15.84100 -O 14.00100 19.69100 4.26000 -H 13.76100 19.32100 3.36000 -H 14.99100 19.85100 4.30000 -O 3.82000 25.62100 4.80000 -H 4.27000 24.72100 4.77000 -H 2.88000 25.51100 5.13000 -O 15.47100 20.84100 17.29100 -H 15.42100 21.21100 18.21100 -H 14.75100 20.15100 17.17100 -O 7.81000 21.26100 17.49100 -H 8.48000 20.65100 17.92100 -H 7.08000 21.45100 18.14100 -O 13.51100 24.52100 14.33100 -H 13.79100 24.09100 15.18100 -H 13.76100 25.48100 14.34100 -O 9.22000 23.65100 8.99000 -H 8.97000 23.56100 8.03000 -H 9.70000 22.83100 9.30000 -O 5.39000 19.26100 5.12000 -H 4.58000 19.27100 5.70000 -H 5.42000 20.09100 4.57000 -O 2.97000 18.97100 1.71000 -H 3.46000 19.81100 1.50000 -H 3.59000 18.32100 2.16000 -O 9.35100 20.98100 4.80000 -H 8.87100 21.39100 4.02000 -H 10.34100 20.96100 4.61000 -O 10.76100 25.45100 14.64100 -H 9.96100 24.84100 14.67100 -H 11.57100 24.92100 14.40100 -O 4.59000 2.31100 7.41000 -H 3.88000 2.04100 8.06000 -H 4.33000 2.03100 6.48000 -O 10.32100 24.11100 0.16000 -H 9.91100 24.93100 -0.23000 -H 10.96100 24.37100 0.89000 -O 0.78000 24.18100 13.86100 -H 1.70000 24.17100 13.45100 -H 0.72000 24.92100 14.53100 -O 5.61000 20.84100 11.47100 -H 5.99000 20.00100 11.84100 -H 4.73000 21.03100 11.91100 -O 8.66000 23.16100 6.42000 -H 8.34000 23.88100 5.80000 -H 8.90000 22.35100 5.89000 -O 10.17100 19.01100 7.53000 -H 9.45100 19.06100 6.84000 -H 9.93100 18.32100 8.22000 -O 14.66100 24.52100 9.92100 -H 14.36100 23.57100 9.99100 -H 15.39100 24.68100 10.58100 -O 18.57100 26.95100 3.77000 -H 18.99100 26.31100 4.41000 -H 18.19100 26.44100 2.99000 -O 13.90100 24.37100 0.78000 -H 13.36100 24.16100 1.59000 -H 14.81100 23.96100 0.87000 -O 5.27000 21.18100 3.28000 -H 5.54000 20.59100 2.53000 -H 5.27000 22.13100 2.97000 -O 9.15000 19.51100 14.02100 -H 9.40000 19.31100 13.07100 -H 9.87000 20.07100 14.44100 -O 4.10000 26.75100 12.51100 -H 4.96000 26.87100 13.01100 -H 3.68000 25.88100 12.78100 -O 12.74100 22.48100 12.62100 -H 12.95100 23.22100 13.26100 -H 11.85100 22.65100 12.19100 -O 5.66000 23.99100 8.65000 -H 5.78000 24.65100 7.91000 -H 6.12000 24.33100 9.48000 -O 15.82100 25.01100 4.72000 -H 15.51100 25.62100 5.45000 -H 16.32100 25.53100 4.03000 -O 4.02000 26.13100 15.98100 -H 4.70000 26.68100 15.51100 -H 4.42000 25.25100 16.25100 -O 15.87100 26.41100 16.70100 -H 14.95100 26.79100 16.65100 -H 16.47100 26.88100 16.05100 -O 10.13100 19.67100 17.70100 -H 10.19100 20.52100 17.18100 -H 10.45100 18.91100 17.13100 -O 5.04000 19.12100 17.40100 -H 4.62000 18.55100 16.70100 -H 4.38000 19.81100 17.72100 -O 5.73000 27.32100 10.29100 -H 6.17000 28.21100 10.20100 -H 5.10000 27.32100 11.06100 -O 13.60100 27.24100 10.45100 -H 12.85100 27.24100 9.79100 -H 13.97100 26.32100 10.54100 -O 12.09100 23.87100 2.75000 -H 12.22100 23.03100 3.29000 -H 11.80100 24.61100 3.35000 -O 3.07000 20.75100 12.31100 -H 2.84000 21.12100 13.21100 -H 2.77000 19.80100 12.25100 -O 7.32000 24.96100 10.64100 -H 7.91000 24.70100 9.88100 -H 7.04000 25.92100 10.53100 -O 17.28100 0.14100 18.54100 -H 16.82100 0.07100 17.65100 -H 16.66100 0.58100 19.20100 -O 3.07000 19.25100 6.18000 -H 2.96000 20.19100 6.51000 -H 3.02000 18.62100 6.95000 -O 16.22100 22.29100 3.74000 -H 16.24100 21.53100 4.38000 -H 15.74100 23.06100 4.14000 -O 10.23100 26.28100 9.66100 -H 10.38100 26.49100 10.62100 -H 9.93100 25.33100 9.57100 -O 15.40100 21.36100 1.25000 -H 14.79100 20.61100 1.48000 -H 15.62100 21.88100 2.08000 -O 3.19000 26.72100 9.49100 -H 4.12000 27.08100 9.54100 -H 3.13000 25.87100 10.01100 -O 3.39000 23.71100 10.06100 -H 2.87000 22.88100 9.89100 -H 4.16000 23.76100 9.42100 -O 11.38100 22.42100 16.78100 -H 10.93100 23.05100 17.42100 -H 12.31100 22.73100 16.61100 -O 11.60100 20.69100 14.77100 -H 11.60100 21.33100 15.54100 -H 11.88100 21.17100 13.94100 -O 10.02100 26.58100 12.38100 -H 10.43100 26.26100 13.24100 -H 9.06100 26.31100 12.35100 -O 0.40000 24.06100 11.14100 -H 1.25000 23.73100 10.73100 -H 0.53000 24.21100 12.12100 -O 1.89000 23.82100 17.22100 -H 2.48000 23.42100 16.52100 -H 1.31000 24.53100 16.81100 -O 8.15000 24.34100 3.25000 -H 8.22000 23.45100 2.79000 -H 7.21000 24.68100 3.17000 -O 16.57100 24.66100 12.06100 -H 16.19100 23.97100 12.68100 -H 17.39100 24.30100 11.62100 -O 6.71000 23.26100 12.69100 -H 6.37000 22.37100 12.39100 -H 6.97000 23.80100 11.89100 -O 4.73000 23.62100 1.91000 -H 5.34000 24.42100 1.95000 -H 3.78000 23.93100 1.98000 -O 9.03000 19.48100 1.33000 -H 9.54000 19.49100 0.47000 -H 9.59000 19.06100 2.04000 -O 17.26100 19.97100 5.23000 -H 17.99100 19.80100 4.56000 -H 16.95100 19.10100 5.61000 -O 12.80100 0.06100 6.72000 -H 13.40100 -0.74900 6.74000 -H 13.20100 0.75100 6.12000 -O 18.41100 20.37100 9.63100 -H 18.80100 19.52100 9.27100 -H 17.43100 20.39100 9.44100 -O 2.63000 21.88100 7.20000 -H 1.84000 22.39100 6.86000 -H 2.54000 21.73100 8.18000 -O 10.14100 20.98100 9.71100 -H 10.06100 20.62100 8.78100 -H 10.12100 20.22100 10.36100 -O 17.09100 22.47100 13.81100 -H 17.82100 23.16100 13.85100 -H 17.37100 21.72100 13.22100 -O 6.72000 20.65100 14.89100 -H 7.62000 20.49100 14.49100 -H 6.80000 20.70100 15.88100 -O 14.40100 27.18100 13.98100 -H 13.83100 27.70100 13.35100 -H 15.36100 27.30100 13.74100 -O 0.72000 20.28100 3.18000 -H 0.55000 21.11100 2.64000 -H 1.62000 19.91100 2.96000 -O 6.13000 27.04100 14.31100 -H 6.69000 27.85100 14.14100 -H 6.72000 26.24100 14.34100 -O 7.16000 24.27100 17.08100 -H 7.35000 24.92100 17.82100 -H 7.76000 23.47100 17.17100 -O 16.74100 27.45100 12.54100 -H 16.47100 26.56100 12.17100 -H 16.75100 28.13100 11.81100 -O 12.25100 21.87100 4.49000 -H 12.90100 21.13100 4.38000 -H 12.45100 22.37100 5.33000 -O 5.94000 26.07100 6.52000 -H 6.44000 26.92100 6.33000 -H 5.06000 26.09100 6.04000 -O 17.77100 22.04100 16.42100 -H 17.60100 22.35100 15.48100 -H 16.93100 21.67100 16.80100 -O 17.30100 0.13100 15.17100 -H 17.68100 1.04100 15.32100 -H 17.22100 -0.03900 14.18100 -O 18.24100 20.54100 12.27100 -H 18.20100 19.64100 12.71100 -H 18.27100 20.43100 11.28100 -O 4.28000 22.86100 5.20000 -H 4.58000 22.14100 4.58000 -H 3.89000 22.46100 6.03000 -O 3.17000 24.09100 12.80100 -H 3.55000 23.50100 13.52100 -H 3.57000 23.83100 11.92100 -O 14.24100 20.76100 11.12100 -H 14.76100 20.11100 11.67100 -H 13.75100 21.39100 11.73100 -O 10.01100 18.96100 11.54100 -H 9.38100 18.24100 11.23100 -H 10.94100 18.60100 11.54100 -O 13.52100 19.14100 1.68000 -H 13.87100 18.73100 0.84000 -H 12.62100 18.76100 1.88000 -O 13.00100 23.15100 6.91000 -H 12.41100 23.95100 6.95000 -H 13.15100 22.80100 7.84000 -O 15.93100 20.83100 8.82000 -H 15.09100 20.82100 9.36000 -H 15.95100 21.66100 8.26000 -O 2.30000 24.90100 19.75100 -H 1.37000 24.88100 20.12100 -H 2.31000 24.51100 18.83100 -O 17.43100 24.80100 27.18100 -H 17.76100 25.74100 27.18100 -H 17.94100 24.26100 27.84100 -O 11.35100 25.65100 25.79100 -H 11.92100 26.43100 25.54100 -H 10.75100 25.91100 26.55100 -O 17.55100 24.69100 20.93100 -H 17.43100 24.56100 19.94100 -H 17.25100 23.88100 21.42100 -O 6.85000 0.91100 25.27100 -H 7.54000 0.75100 25.97100 -H 6.12000 1.48100 25.65100 -O 2.40000 1.70100 27.48100 -H 2.54000 0.86100 28.00100 -H 1.85000 2.34100 28.03100 -O 8.09000 18.66100 23.64100 -H 8.49000 19.57100 23.55100 -H 7.09000 18.74100 23.70100 -O 5.98000 25.91100 21.32100 -H 6.22000 26.60100 20.64100 -H 5.20000 26.24100 21.86100 -O 12.81100 22.07100 0.23100 -H 11.95100 21.57100 0.10100 -H 13.43100 21.53100 0.79100 -O 10.41100 25.63100 22.91100 -H 10.67100 25.59100 23.87100 -H 9.56100 25.12100 22.77100 -O 0.76000 26.73100 26.51100 -H 1.75000 26.61100 26.60100 -H 0.52000 27.68100 26.72100 -O 8.65000 22.10100 20.57100 -H 9.24000 22.73100 20.08100 -H 8.84000 21.16100 20.28100 -O 5.50000 20.58100 27.47100 -H 5.45000 20.53100 28.47100 -H 5.52000 21.54100 27.18100 -O 14.58100 25.97100 25.90100 -H 14.53100 25.32100 26.65100 -H 15.38100 26.56100 26.03100 -O 14.00100 19.69100 22.88100 -H 13.76100 19.32100 21.98100 -H 14.99100 19.85100 22.92100 -O 3.82000 25.62100 23.42100 -H 4.27000 24.72100 23.39100 -H 2.88000 25.51100 23.75100 -O 6.14000 19.84100 19.79100 -H 7.12000 19.62100 19.86100 -H 5.83000 19.67100 18.86100 -O 5.39000 19.26100 23.74100 -H 4.58000 19.27100 24.32100 -H 5.42000 20.09100 23.19100 -O 2.97000 18.97100 20.33100 -H 3.46000 19.81100 20.12100 -H 3.59000 18.32100 20.78100 -O 9.35100 20.98100 23.42100 -H 8.87100 21.39100 22.64100 -H 10.34100 20.96100 23.23100 -O 4.59000 2.31100 26.03100 -H 3.88000 2.04100 26.68100 -H 4.33000 2.03100 25.10100 -O 10.32100 24.11100 18.78100 -H 9.91100 24.93100 18.39100 -H 10.96100 24.37100 19.51100 -O 8.66000 23.16100 25.04100 -H 8.34000 23.88100 24.42100 -H 8.90000 22.35100 24.51100 -O 10.17100 19.01100 26.15100 -H 9.45100 19.06100 25.46100 -H 9.93100 18.32100 26.84100 -O 18.57100 26.95100 22.39100 -H 18.99100 26.31100 23.03100 -H 18.19100 26.44100 21.61100 -O 13.90100 24.37100 19.40100 -H 13.36100 24.16100 20.21100 -H 14.81100 23.96100 19.49100 -O 5.27000 21.18100 21.90100 -H 5.54000 20.59100 21.15100 -H 5.27000 22.13100 21.59100 -O 5.66000 23.99100 27.27100 -H 5.78000 24.65100 26.53100 -H 6.12000 24.33100 28.10100 -O 15.82100 25.01100 23.34100 -H 15.51100 25.62100 24.07100 -H 16.32100 25.53100 22.65100 -O 12.09100 23.87100 21.37100 -H 12.22100 23.03100 21.91100 -H 11.80100 24.61100 21.97100 -O 3.07000 19.25100 24.80100 -H 2.96000 20.19100 25.13100 -H 3.02000 18.62100 25.57100 -O 16.22100 22.29100 22.36100 -H 16.24100 21.53100 23.00100 -H 15.74100 23.06100 22.76100 -O 15.40100 21.36100 19.87100 -H 14.79100 20.61100 20.10100 -H 15.62100 21.88100 20.70100 -O 8.15000 24.34100 21.87100 -H 8.22000 23.45100 21.41100 -H 7.21000 24.68100 21.79100 -O 4.73000 23.62100 20.53100 -H 5.34000 24.42100 20.57100 -H 3.78000 23.93100 20.60100 -O 9.03000 19.48100 19.95100 -H 9.54000 19.49100 19.09100 -H 9.59000 19.06100 20.66100 -O 17.26100 19.97100 23.85100 -H 17.99100 19.80100 23.18100 -H 16.95100 19.10100 24.23100 -O 12.80100 0.06100 25.34100 -H 13.40100 -0.74900 25.36100 -H 13.20100 0.75100 24.74100 -O 18.41100 20.37100 0.42100 -H 18.80100 19.52100 0.06100 -H 17.43100 20.39100 0.23100 -O 2.63000 21.88100 25.82100 -H 1.84000 22.39100 25.48100 -H 2.54000 21.73100 26.80100 -O 9.16000 27.72100 21.53100 -H 9.79000 28.10100 20.85100 -H 9.56000 26.89100 21.92100 -O 0.75000 22.07100 18.95100 -H -0.17000 21.79100 18.66100 -H 1.06000 22.84100 18.39100 -O 0.72000 20.28100 21.80100 -H 0.55000 21.11100 21.26100 -H 1.62000 19.91100 21.58100 -O 12.25100 21.87100 23.11100 -H 12.90100 21.13100 23.00100 -H 12.45100 22.37100 23.95100 -O 5.94000 26.07100 25.14100 -H 6.44000 26.92100 24.95100 -H 5.06000 26.09100 24.66100 -O 4.28000 22.86100 23.82100 -H 4.58000 22.14100 23.20100 -H 3.89000 22.46100 24.65100 -O 13.52100 19.14100 20.30100 -H 13.87100 18.73100 19.46100 -H 12.62100 18.76100 20.50100 -O 13.00100 23.15100 25.53100 -H 12.41100 23.95100 25.57100 -H 13.15100 22.80100 26.46100 -O 20.92100 6.28000 1.13000 -H 19.99100 6.26000 1.50000 -H 20.93100 5.89000 0.21000 -O 20.87100 2.75000 9.96100 -H 21.22100 2.58000 10.88100 -H 19.99100 2.30000 9.84100 -O 18.81100 3.68000 6.47000 -H 17.99100 4.11000 6.86000 -H 18.53100 2.95000 5.84000 -O 24.31100 12.75100 11.65100 -H 23.38100 12.68100 11.28100 -H 24.42100 13.64100 12.09100 -O 27.12100 7.98000 18.23100 -H 27.08100 8.74000 18.88100 -H 27.34100 8.34000 17.32100 -O 25.47100 10.12100 6.65000 -H 26.16100 9.96100 7.35000 -H 24.74100 10.69100 7.03000 -O 25.48100 11.61100 18.03100 -H 26.08100 12.40100 18.17100 -H 24.62100 11.92100 17.62100 -O 21.97100 14.35100 10.61100 -H 21.19100 14.04100 10.08100 -H 22.55100 14.93100 10.04100 -O 23.00100 3.92000 14.99100 -H 23.82100 3.36000 15.08100 -H 22.19100 3.34000 15.03100 -O 20.93100 17.13100 4.83000 -H 21.27100 17.90100 5.37000 -H 21.37100 17.13100 3.93000 -O 20.92100 14.34100 5.38000 -H 20.66100 15.30100 5.38000 -H 20.21100 13.80100 5.83000 -O 21.02100 10.91100 8.86000 -H 21.16100 10.07100 9.38000 -H 20.47100 11.55100 9.41000 -O 24.82100 17.86100 14.39100 -H 23.90100 17.69100 14.74100 -H 25.10100 18.78100 14.65100 -O 24.68100 9.64100 1.23000 -H 24.75100 10.48100 0.69000 -H 25.14100 9.77100 2.11000 -O 19.84100 6.43000 5.63000 -H 19.39100 5.55000 5.80000 -H 19.83100 6.97000 6.47000 -O 18.89100 15.96100 1.17000 -H 18.70100 15.00100 1.38000 -H 18.56100 16.54100 1.92000 -O 19.75100 7.37000 15.97100 -H 20.63100 7.24000 16.42100 -H 19.62100 8.34000 15.75100 -O 24.75100 13.65100 7.26000 -H 24.26100 12.78100 7.35000 -H 24.52100 14.08100 6.39000 -O 26.71100 0.04000 5.02000 -H 27.11100 0.95000 4.93000 -H 25.71100 0.12000 5.08000 -O 20.59100 9.76100 12.64100 -H 21.48100 9.31100 12.50100 -H 19.86100 9.11100 12.45100 -O 21.79100 2.51000 18.01100 -H 22.50100 3.22000 18.07100 -H 20.91100 2.90000 18.29100 -O 24.60100 7.29000 2.70000 -H 24.84100 7.98000 2.02000 -H 23.82100 7.62000 3.24000 -O 26.69100 6.05000 14.65100 -H 26.22100 6.02000 15.54100 -H 26.18100 5.50000 13.99100 -O 0.52100 8.90000 15.72100 -H 0.98100 8.06000 15.43100 -H -0.03900 9.24000 14.97100 -O 1.20100 7.01000 4.29000 -H 1.46100 6.97000 5.25000 -H 0.35100 6.50000 4.15000 -O 19.38100 8.11000 7.89000 -H 20.37100 7.99000 7.98000 -H 19.14100 9.06000 8.10000 -O 19.92100 18.21100 15.71100 -H 19.82100 18.06100 16.70100 -H 19.06100 18.57100 15.35100 -O 24.12100 1.96000 8.85000 -H 24.07100 1.91000 9.85000 -H 24.14100 2.92000 8.56000 -O 0.87100 14.56100 4.77000 -H 0.41100 15.28100 4.25000 -H 0.83100 14.76100 5.75000 -O 22.13100 18.01100 8.53000 -H 22.63100 17.15100 8.59000 -H 22.78100 18.78100 8.50000 -O 21.83100 9.43100 2.42000 -H 22.65100 9.82100 2.00000 -H 21.56100 8.61100 1.93000 -O 23.23100 12.66100 17.27100 -H 22.73100 12.67100 16.41100 -H 22.60100 12.48100 18.03100 -O 1.90100 17.76100 2.37000 -H 1.30100 17.14100 2.87000 -H 2.21100 17.32100 1.52000 -O 20.64100 2.85000 14.98100 -H 19.84100 3.45000 14.85100 -H 20.54100 2.36000 15.84100 -O 23.26100 17.43100 3.23000 -H 23.59100 17.82100 4.09000 -H 24.02100 17.36100 2.58000 -O 21.11100 17.85100 12.41100 -H 21.68100 17.20100 12.91100 -H 20.95100 17.52100 11.48100 -O 22.44100 7.00000 4.80000 -H 22.89100 6.10000 4.77000 -H 21.50100 6.89000 5.13000 -O 24.76100 1.22000 1.17000 -H 25.74100 1.00000 1.24000 -H 24.45100 1.05000 0.24000 -O 26.43100 2.64000 17.49100 -H 27.10100 2.03000 17.92100 -H 25.70100 2.83000 18.14100 -O 26.65100 14.02100 9.24000 -H 27.55100 14.16100 8.82000 -H 25.94100 14.04100 8.53000 -O 0.01100 5.03000 8.99000 -H -0.23900 4.94000 8.03000 -H 0.49100 4.21000 9.30000 -O 24.01100 0.64000 5.12000 -H 23.20100 0.65000 5.70000 -H 24.04100 1.47000 4.57000 -O 21.59100 0.35000 1.71000 -H 22.08100 1.19000 1.50000 -H 22.21100 -0.30000 2.16000 -O 0.14100 2.36000 4.80000 -H -0.33900 2.77000 4.02000 -H 1.13100 2.34000 4.61000 -O 23.21100 11.52100 7.41000 -H 22.50100 11.25100 8.06000 -H 22.95100 11.24100 6.48000 -O 19.40100 5.56000 13.86100 -H 20.32100 5.55000 13.45100 -H 19.34100 6.30000 14.53100 -O 24.23100 2.22000 11.47100 -H 24.61100 1.38000 11.84100 -H 23.35100 2.41000 11.91100 -O 27.28100 4.54000 6.42000 -H 26.96100 5.26000 5.80000 -H 27.52100 3.73000 5.89000 -O 23.50100 13.85100 1.74000 -H 22.63100 13.70100 2.21000 -H 23.33100 14.11100 0.79000 -O 23.89100 2.56000 3.28000 -H 24.16100 1.97000 2.53000 -H 23.89100 3.51000 2.97000 -O 27.77100 0.89000 14.02100 -H 28.02100 0.69000 13.07100 -H 28.49100 1.45000 14.44100 -O 22.72100 8.13000 12.51100 -H 23.58100 8.25000 13.01100 -H 22.30100 7.26000 12.78100 -O 19.26100 15.64100 13.31100 -H 18.80100 16.46100 12.97100 -H 20.24100 15.83100 13.40100 -O 22.29100 11.00100 5.01000 -H 22.22100 11.83100 4.45000 -H 22.33100 10.20100 4.41000 -O 24.28100 5.37000 8.65000 -H 24.40100 6.03000 7.91000 -H 24.74100 5.71000 9.48000 -O 22.16100 15.10100 13.29100 -H 21.95100 14.66100 12.42100 -H 23.13100 15.36100 13.32100 -O 22.64100 7.51000 15.98100 -H 23.32100 8.06000 15.51100 -H 23.04100 6.63000 16.25100 -O 23.66100 0.50000 17.40100 -H 23.24100 -0.07000 16.70100 -H 23.00100 1.19000 17.72100 -O 24.35100 8.70000 10.29100 -H 24.79100 9.59000 10.20100 -H 23.72100 8.70000 11.06100 -O 21.69100 2.13000 12.31100 -H 21.46100 2.50000 13.21100 -H 21.39100 1.18000 12.25100 -O 18.99100 13.10100 12.82100 -H 19.52100 12.61100 13.50100 -H 19.21100 14.08100 12.87100 -O 25.94100 6.34000 10.64100 -H 26.53100 6.08000 9.88100 -H 25.66100 7.30000 10.53100 -O 21.69100 0.63000 6.18000 -H 21.58100 1.57000 6.51000 -H 21.64100 0.00000 6.95000 -O 1.02100 7.66000 9.66100 -H 1.17100 7.87000 10.62100 -H 0.72100 6.71000 9.57100 -O 0.59100 15.73100 17.00100 -H 0.39100 16.17100 16.12100 -H 0.98100 14.82100 16.84100 -O 22.12100 8.98000 18.04100 -H 22.88100 9.42000 18.52100 -H 22.47100 8.51000 17.22100 -O 21.81100 8.10000 9.49100 -H 22.74100 8.46000 9.54100 -H 21.75100 7.25000 10.01100 -O 22.01100 5.09000 10.06100 -H 21.49100 4.26000 9.89100 -H 22.78100 5.14000 9.42100 -O 18.70100 13.26100 2.00000 -H 17.77100 13.47100 1.69000 -H 18.80100 12.27100 2.13000 -O 19.50100 18.01100 9.27000 -H 19.08100 17.15100 9.00000 -H 20.44100 18.04100 8.93000 -O 23.66100 15.68100 9.10000 -H 24.32100 16.42100 9.19000 -H 24.10100 14.89100 8.68000 -O 0.81100 7.96000 12.38100 -H 1.22100 7.64000 13.24100 -H -0.14900 7.69000 12.35100 -O 19.02100 5.44000 11.14100 -H 19.87100 5.11000 10.73100 -H 19.15100 5.59000 12.12100 -O 20.51100 5.20000 17.22100 -H 21.10100 4.80000 16.52100 -H 19.93100 5.91000 16.81100 -O 26.77100 5.72000 3.25000 -H 26.84100 4.83000 2.79000 -H 25.83100 6.06000 3.17000 -O 21.14100 15.64100 17.44100 -H 20.84100 16.21100 18.20100 -H 21.07100 14.67100 17.70100 -O 25.33100 4.64000 12.69100 -H 24.99100 3.75000 12.39100 -H 25.59100 5.18000 11.89100 -O 0.09100 16.78100 14.65100 -H -0.14900 16.60100 13.70100 -H 0.39100 17.72100 14.75100 -O 23.35100 5.00000 1.91000 -H 23.96100 5.80000 1.95000 -H 22.40100 5.31000 1.98000 -O 20.21100 11.37100 14.66100 -H 20.43100 10.76100 15.42100 -H 20.31100 10.88100 13.80100 -O 19.45100 12.58100 10.22100 -H 19.40100 12.57100 11.22100 -H 18.62100 12.17100 9.84100 -O 25.50100 16.62100 17.16100 -H 24.94100 17.43100 17.25100 -H 26.02100 16.66100 16.30100 -O 27.65100 0.86000 1.33000 -H 28.16100 0.87000 0.47000 -H 28.21100 0.44000 2.04000 -O 19.92100 17.94100 18.51100 -H 19.51100 17.20100 19.04100 -H 20.56100 18.45100 19.09100 -O 26.92100 12.73100 14.22100 -H 26.87100 13.06100 15.17100 -H 26.06100 12.92100 13.76100 -O 25.34100 16.16100 1.54000 -H 25.43100 16.26100 0.55000 -H 24.94100 15.27100 1.75000 -O 21.25100 3.26000 7.20000 -H 20.46100 3.77000 6.86000 -H 21.16100 3.11000 8.18000 -O 26.84100 10.02100 13.72100 -H 27.24100 10.01100 12.80100 -H 26.94100 10.94100 14.12100 -O 22.34100 12.88100 14.90100 -H 22.21100 13.81100 14.56100 -H 21.50100 12.36100 14.77100 -O 0.93100 2.36000 9.71100 -H 0.85100 2.00000 8.78100 -H 0.91100 1.60000 10.36100 -O 24.52100 14.87100 4.91000 -H 24.94100 14.29100 4.21000 -H 24.08100 15.66100 4.47000 -O 21.17100 13.48100 2.90000 -H 20.21100 13.49100 2.63000 -H 21.29100 14.01100 3.74000 -O 19.67100 10.13100 17.26100 -H 18.90100 9.80100 17.80100 -H 20.52100 9.83100 17.68100 -O 25.34100 2.03000 14.89100 -H 26.24100 1.87000 14.49100 -H 25.42100 2.08000 15.88100 -O 19.34100 1.66000 3.18000 -H 19.17100 2.49000 2.64000 -H 20.24100 1.29000 2.96000 -O 24.75100 8.42000 14.31100 -H 25.31100 9.23000 14.14100 -H 25.34100 7.62000 14.34100 -O 25.78100 5.65000 17.08100 -H 25.97100 6.30000 17.82100 -H 26.38100 4.85000 17.17100 -O 22.52100 17.41100 15.60100 -H 21.61100 17.82100 15.58100 -H 22.45100 16.47100 15.92100 -O 24.56100 7.45000 6.52000 -H 25.06100 8.30000 6.33000 -H 23.68100 7.47000 6.04000 -O 27.21100 13.74100 0.16000 -H 26.75100 13.89100 1.04000 -H 27.65100 14.59100 -0.14000 -O 25.23100 17.90100 9.53100 -H 24.77100 18.78100 9.40100 -H 26.22100 18.02100 9.46100 -O 27.21100 9.56100 8.61000 -H 27.75100 8.87100 9.09000 -H 26.89100 10.25100 9.27000 -O 20.83100 13.14100 18.44100 -H 20.18100 12.41100 18.23100 -H 20.87100 13.28100 19.42100 -O 19.41100 12.40100 6.53000 -H 19.40100 11.93100 7.41000 -H 20.23100 12.12100 6.02000 -O 25.34100 13.91100 16.24100 -H 24.56100 13.41100 16.62100 -H 25.31100 14.86100 16.55100 -O 22.90100 4.24000 5.20000 -H 23.20100 3.52000 4.58000 -H 22.51100 3.84000 6.03000 -O 21.79100 5.47000 12.80100 -H 22.17100 4.88000 13.52100 -H 22.19100 5.21000 11.92100 -O 26.74100 15.86100 6.87000 -H 27.06100 15.96100 5.93000 -H 25.95100 15.24100 6.89000 -O 0.80100 0.34000 11.54100 -H 0.17100 -0.38000 11.23100 -H 1.73100 -0.02000 11.54100 -O 26.32100 13.30100 3.01000 -H 25.86100 12.43100 3.18000 -H 27.23100 13.27100 3.42000 -O 24.80100 15.67100 12.84100 -H 24.75100 16.49100 13.41100 -H 25.69100 15.64100 12.39100 -O 19.01100 10.77100 3.00000 -H 20.00100 10.66100 2.91000 -H 18.61100 9.91100 3.32000 -O 27.37100 16.46100 3.37000 -H 26.60100 16.11100 2.83000 -H 27.05100 17.17100 3.99000 -O 20.92100 6.28000 19.75100 -H 19.99100 6.26000 20.12100 -H 20.93100 5.89000 18.83100 -O 20.87100 2.75000 0.75100 -H 21.22100 2.58000 1.67100 -H 19.99100 2.30000 0.63100 -O 18.81100 3.68000 25.09100 -H 17.99100 4.11000 25.48100 -H 18.53100 2.95000 24.46100 -O 26.30100 11.44100 1.02100 -H 25.52100 11.61100 1.62100 -H 26.64100 12.31100 0.66100 -O 25.47100 10.12100 25.27100 -H 26.16100 9.96100 25.97100 -H 24.74100 10.69100 25.65100 -O 20.93100 17.13100 23.45100 -H 21.27100 17.90100 23.99100 -H 21.37100 17.13100 22.55100 -O 20.92100 14.34100 24.00100 -H 20.66100 15.30100 24.00100 -H 20.21100 13.80100 24.45100 -O 21.02100 10.91100 27.48100 -H 21.16100 10.07100 28.00100 -H 20.47100 11.55100 28.03100 -O 24.68100 9.64100 19.85100 -H 24.75100 10.48100 19.31100 -H 25.14100 9.77100 20.73100 -O 19.84100 6.43000 24.25100 -H 19.39100 5.55000 24.42100 -H 19.83100 6.97000 25.09100 -O 18.89100 15.96100 19.79100 -H 18.70100 15.00100 20.00100 -H 18.56100 16.54100 20.54100 -O 24.75100 13.65100 25.88100 -H 24.26100 12.78100 25.97100 -H 24.52100 14.08100 25.01100 -O 26.71100 0.04000 23.64100 -H 27.11100 0.95000 23.55100 -H 25.71100 0.12000 23.70100 -O 24.60100 7.29000 21.32100 -H 24.84100 7.98000 20.64100 -H 23.82100 7.62000 21.86100 -O 1.20100 7.01000 22.91100 -H 1.46100 6.97000 23.87100 -H 0.35100 6.50000 22.77100 -O 19.38100 8.11000 26.51100 -H 20.37100 7.99000 26.60100 -H 19.14100 9.06000 26.72100 -O 0.87100 14.56100 23.39100 -H 0.41100 15.28100 22.87100 -H 0.83100 14.76100 24.37100 -O 22.13100 18.01100 27.15100 -H 22.63100 17.15100 27.21100 -H 22.78100 18.78100 27.12100 -O 21.83100 9.43100 21.04100 -H 22.65100 9.82100 20.62100 -H 21.56100 8.61100 20.55100 -O 1.90100 17.76100 20.99100 -H 1.30100 17.14100 21.49100 -H 2.21100 17.32100 20.14100 -O 23.26100 17.43100 21.85100 -H 23.59100 17.82100 22.71100 -H 24.02100 17.36100 21.20100 -O 22.44100 7.00000 23.42100 -H 22.89100 6.10000 23.39100 -H 21.50100 6.89000 23.75100 -O 24.76100 1.22000 19.79100 -H 25.74100 1.00000 19.86100 -H 24.45100 1.05000 18.86100 -O 0.01100 5.03000 27.61100 -H -0.23900 4.94000 26.65100 -H 0.49100 4.21000 27.92100 -O 24.01100 0.64000 23.74100 -H 23.20100 0.65000 24.32100 -H 24.04100 1.47000 23.19100 -O 21.59100 0.35000 20.33100 -H 22.08100 1.19000 20.12100 -H 22.21100 -0.30000 20.78100 -O 0.14100 2.36000 23.42100 -H -0.33900 2.77000 22.64100 -H 1.13100 2.34000 23.23100 -O 23.21100 11.52100 26.03100 -H 22.50100 11.25100 26.68100 -H 22.95100 11.24100 25.10100 -O 27.28100 4.54000 25.04100 -H 26.96100 5.26000 24.42100 -H 27.52100 3.73000 24.51100 -O 23.50100 13.85100 20.36100 -H 22.63100 13.70100 20.83100 -H 23.33100 14.11100 19.41100 -O 23.89100 2.56000 21.90100 -H 24.16100 1.97000 21.15100 -H 23.89100 3.51000 21.59100 -O 22.29100 11.00100 23.63100 -H 22.22100 11.83100 23.07100 -H 22.33100 10.20100 23.03100 -O 24.28100 5.37000 27.27100 -H 24.40100 6.03000 26.53100 -H 24.74100 5.71000 28.10100 -O 21.69100 0.63000 24.80100 -H 21.58100 1.57000 25.13100 -H 21.64100 0.00000 25.57100 -O 1.02100 7.66000 0.45100 -H 1.17100 7.87000 1.41100 -H 0.72100 6.71000 0.36100 -O 18.70100 13.26100 20.62100 -H 17.77100 13.47100 20.31100 -H 18.80100 12.27100 20.75100 -O 19.50100 18.01100 0.06100 -H 19.08100 17.15100 -0.20900 -H 20.44100 18.04100 -0.27900 -O 26.77100 5.72000 21.87100 -H 26.84100 4.83000 21.41100 -H 25.83100 6.06000 21.79100 -O 23.35100 5.00000 20.53100 -H 23.96100 5.80000 20.57100 -H 22.40100 5.31000 20.60100 -O 25.34100 16.16100 20.16100 -H 25.43100 16.26100 19.17100 -H 24.94100 15.27100 20.37100 -O 21.25100 3.26000 25.82100 -H 20.46100 3.77000 25.48100 -H 21.16100 3.11000 26.80100 -O 24.52100 14.87100 23.53100 -H 24.94100 14.29100 22.83100 -H 24.08100 15.66100 23.09100 -O 21.17100 13.48100 21.52100 -H 20.21100 13.49100 21.25100 -H 21.29100 14.01100 22.36100 -O 19.37100 3.45000 18.95100 -H 18.45100 3.17000 18.66100 -H 19.68100 4.22000 18.39100 -O 19.34100 1.66000 21.80100 -H 19.17100 2.49000 21.26100 -H 20.24100 1.29000 21.58100 -O 24.56100 7.45000 25.14100 -H 25.06100 8.30000 24.95100 -H 23.68100 7.47000 24.66100 -O 27.21100 13.74100 18.78100 -H 26.75100 13.89100 19.66100 -H 27.65100 14.59100 18.48100 -O 25.23100 17.90100 0.32100 -H 24.77100 18.78100 0.19100 -H 26.22100 18.02100 0.25100 -O 27.21100 9.56100 27.23100 -H 27.75100 8.87100 27.71100 -H 26.89100 10.25100 27.89100 -O 1.62100 9.84100 19.49100 -H 1.39100 10.37100 18.67100 -H 2.43100 9.28100 19.30100 -O 19.41100 12.40100 25.15100 -H 19.40100 11.93100 26.03100 -H 20.23100 12.12100 24.64100 -O 22.90100 4.24000 23.82100 -H 23.20100 3.52000 23.20100 -H 22.51100 3.84000 24.65100 -O 26.74100 15.86100 25.49100 -H 27.06100 15.96100 24.55100 -H 25.95100 15.24100 25.51100 -O 26.32100 13.30100 21.63100 -H 25.86100 12.43100 21.80100 -H 27.23100 13.27100 22.04100 -O 19.01100 10.77100 21.62100 -H 20.00100 10.66100 21.53100 -H 18.61100 9.91100 21.94100 -O 27.37100 16.46100 21.99100 -H 26.60100 16.11100 21.45100 -H 27.05100 17.17100 22.61100 -O 20.92100 24.90100 1.13000 -H 19.99100 24.88100 1.50000 -H 20.93100 24.51100 0.21000 -O 20.87100 21.37100 9.96100 -H 21.22100 21.20100 10.88100 -H 19.99100 20.92100 9.84100 -O 18.81100 22.30100 6.47000 -H 17.99100 22.73100 6.86000 -H 18.53100 21.57100 5.84000 -O 27.12100 26.60100 18.23100 -H 27.08100 27.36100 18.88100 -H 27.34100 26.96100 17.32100 -O 25.47100 0.91100 6.65000 -H 26.16100 0.75100 7.35000 -H 24.74100 1.48100 7.03000 -O 23.00100 22.54100 14.99100 -H 23.82100 21.98100 15.08100 -H 22.19100 21.96100 15.03100 -O 19.84100 25.05100 5.63000 -H 19.39100 24.17100 5.80000 -H 19.83100 25.59100 6.47000 -O 19.75100 25.99100 15.97100 -H 20.63100 25.86100 16.42100 -H 19.62100 26.96100 15.75100 -O 26.71100 18.66100 5.02000 -H 27.11100 19.57100 4.93000 -H 25.71100 18.74100 5.08000 -O 21.79100 21.13100 18.01100 -H 22.50100 21.84100 18.07100 -H 20.91100 21.52100 18.29100 -O 24.60100 25.91100 2.70000 -H 24.84100 26.60100 2.02000 -H 23.82100 26.24100 3.24000 -O 26.69100 24.67100 14.65100 -H 26.22100 24.64100 15.54100 -H 26.18100 24.12100 13.99100 -O 1.20100 25.63100 4.29000 -H 1.46100 25.59100 5.25000 -H 0.35100 25.12100 4.15000 -O 19.38100 26.73100 7.89000 -H 20.37100 26.61100 7.98000 -H 19.14100 27.68100 8.10000 -O 24.12100 20.58100 8.85000 -H 24.07100 20.53100 9.85000 -H 24.14100 21.54100 8.56000 -O 20.64100 21.47100 14.98100 -H 19.84100 22.07100 14.85100 -H 20.54100 20.98100 15.84100 -O 22.44100 25.62100 4.80000 -H 22.89100 24.72100 4.77000 -H 21.50100 25.51100 5.13000 -O 26.43100 21.26100 17.49100 -H 27.10100 20.65100 17.92100 -H 25.70100 21.45100 18.14100 -O 0.01100 23.65100 8.99000 -H -0.23900 23.56100 8.03000 -H 0.49100 22.83100 9.30000 -O 24.01100 19.26100 5.12000 -H 23.20100 19.27100 5.70000 -H 24.04100 20.09100 4.57000 -O 21.59100 18.97100 1.71000 -H 22.08100 19.81100 1.50000 -H 22.21100 18.32100 2.16000 -O 0.14100 20.98100 4.80000 -H -0.33900 21.39100 4.02000 -H 1.13100 20.96100 4.61000 -O 23.21100 2.31100 7.41000 -H 22.50100 2.04100 8.06000 -H 22.95100 2.03100 6.48000 -O 19.40100 24.18100 13.86100 -H 20.32100 24.17100 13.45100 -H 19.34100 24.92100 14.53100 -O 24.23100 20.84100 11.47100 -H 24.61100 20.00100 11.84100 -H 23.35100 21.03100 11.91100 -O 27.28100 23.16100 6.42000 -H 26.96100 23.88100 5.80000 -H 27.52100 22.35100 5.89000 -O 0.96100 19.01100 7.53000 -H 0.24100 19.06100 6.84000 -H 0.72100 18.32100 8.22000 -O 23.89100 21.18100 3.28000 -H 24.16100 20.59100 2.53000 -H 23.89100 22.13100 2.97000 -O 27.77100 19.51100 14.02100 -H 28.02100 19.31100 13.07100 -H 28.49100 20.07100 14.44100 -O 22.72100 26.75100 12.51100 -H 23.58100 26.87100 13.01100 -H 22.30100 25.88100 12.78100 -O 24.28100 23.99100 8.65000 -H 24.40100 24.65100 7.91000 -H 24.74100 24.33100 9.48000 -O 22.64100 26.13100 15.98100 -H 23.32100 26.68100 15.51100 -H 23.04100 25.25100 16.25100 -O 0.92100 19.67100 17.70100 -H 0.98100 20.52100 17.18100 -H 1.24100 18.91100 17.13100 -O 23.66100 19.12100 17.40100 -H 23.24100 18.55100 16.70100 -H 23.00100 19.81100 17.72100 -O 24.35100 27.32100 10.29100 -H 24.79100 28.21100 10.20100 -H 23.72100 27.32100 11.06100 -O 21.69100 20.75100 12.31100 -H 21.46100 21.12100 13.21100 -H 21.39100 19.80100 12.25100 -O 25.94100 24.96100 10.64100 -H 26.53100 24.70100 9.88100 -H 25.66100 25.92100 10.53100 -O 21.69100 19.25100 6.18000 -H 21.58100 20.19100 6.51000 -H 21.64100 18.62100 6.95000 -O 1.02100 26.28100 9.66100 -H 1.17100 26.49100 10.62100 -H 0.72100 25.33100 9.57100 -O 21.81100 26.72100 9.49100 -H 22.74100 27.08100 9.54100 -H 21.75100 25.87100 10.01100 -O 22.01100 23.71100 10.06100 -H 21.49100 22.88100 9.89100 -H 22.78100 23.76100 9.42100 -O 23.73100 22.77100 18.08100 -H 23.55100 23.22100 18.96100 -H 24.15100 23.42100 17.45100 -O 0.81100 26.58100 12.38100 -H 1.22100 26.26100 13.24100 -H -0.14900 26.31100 12.35100 -O 19.02100 24.06100 11.14100 -H 19.87100 23.73100 10.73100 -H 19.15100 24.21100 12.12100 -O 20.51100 23.82100 17.22100 -H 21.10100 23.42100 16.52100 -H 19.93100 24.53100 16.81100 -O 26.77100 24.34100 3.25000 -H 26.84100 23.45100 2.79000 -H 25.83100 24.68100 3.17000 -O 25.33100 23.26100 12.69100 -H 24.99100 22.37100 12.39100 -H 25.59100 23.80100 11.89100 -O 23.35100 23.62100 1.91000 -H 23.96100 24.42100 1.95000 -H 22.40100 23.93100 1.98000 -O 27.65100 19.48100 1.33000 -H 28.16100 19.49100 0.47000 -H 28.21100 19.06100 2.04000 -O 21.25100 21.88100 7.20000 -H 20.46100 22.39100 6.86000 -H 21.16100 21.73100 8.18000 -O 1.18100 1.77100 6.96000 -H 0.48100 1.30100 7.50000 -H 1.77100 1.09100 6.53000 -O 0.93100 20.98100 9.71100 -H 0.85100 20.62100 8.78100 -H 0.91100 20.22100 10.36100 -O 19.67100 0.92100 17.26100 -H 18.90100 0.59100 17.80100 -H 20.52100 0.62100 17.68100 -O 25.34100 20.65100 14.89100 -H 26.24100 20.49100 14.49100 -H 25.42100 20.70100 15.88100 -O 19.34100 20.28100 3.18000 -H 19.17100 21.11100 2.64000 -H 20.24100 19.91100 2.96000 -O 24.75100 27.04100 14.31100 -H 25.31100 27.85100 14.14100 -H 25.34100 26.24100 14.34100 -O 25.78100 24.27100 17.08100 -H 25.97100 24.92100 17.82100 -H 26.38100 23.47100 17.17100 -O 24.56100 26.07100 6.52000 -H 25.06100 26.92100 6.33000 -H 23.68100 26.09100 6.04000 -O 27.21100 0.35100 8.61000 -H 27.75100 -0.33900 9.09000 -H 26.89100 1.04100 9.27000 -O 22.90100 22.86100 5.20000 -H 23.20100 22.14100 4.58000 -H 22.51100 22.46100 6.03000 -O 21.79100 24.09100 12.80100 -H 22.17100 23.50100 13.52100 -H 22.19100 23.83100 11.92100 -O 0.80100 18.96100 11.54100 -H 0.17100 18.24100 11.23100 -H 1.73100 18.60100 11.54100 -O 20.92100 24.90100 19.75100 -H 19.99100 24.88100 20.12100 -H 20.93100 24.51100 18.83100 -O 20.87100 21.37100 0.75100 -H 21.22100 21.20100 1.67100 -H 19.99100 20.92100 0.63100 -O 18.81100 22.30100 25.09100 -H 17.99100 22.73100 25.48100 -H 18.53100 21.57100 24.46100 -O 25.47100 0.91100 25.27100 -H 26.16100 0.75100 25.97100 -H 24.74100 1.48100 25.65100 -O 19.84100 25.05100 24.25100 -H 19.39100 24.17100 24.42100 -H 19.83100 25.59100 25.09100 -O 26.71100 18.66100 23.64100 -H 27.11100 19.57100 23.55100 -H 25.71100 18.74100 23.70100 -O 24.60100 25.91100 21.32100 -H 24.84100 26.60100 20.64100 -H 23.82100 26.24100 21.86100 -O 1.20100 25.63100 22.91100 -H 1.46100 25.59100 23.87100 -H 0.35100 25.12100 22.77100 -O 19.38100 26.73100 26.51100 -H 20.37100 26.61100 26.60100 -H 19.14100 27.68100 26.72100 -O 24.12100 20.58100 27.47100 -H 24.07100 20.53100 28.47100 -H 24.14100 21.54100 27.18100 -O 22.44100 25.62100 23.42100 -H 22.89100 24.72100 23.39100 -H 21.50100 25.51100 23.75100 -O 24.76100 19.84100 19.79100 -H 25.74100 19.62100 19.86100 -H 24.45100 19.67100 18.86100 -O 0.01100 23.65100 27.61100 -H -0.23900 23.56100 26.65100 -H 0.49100 22.83100 27.92100 -O 24.01100 19.26100 23.74100 -H 23.20100 19.27100 24.32100 -H 24.04100 20.09100 23.19100 -O 21.59100 18.97100 20.33100 -H 22.08100 19.81100 20.12100 -H 22.21100 18.32100 20.78100 -O 0.14100 20.98100 23.42100 -H -0.33900 21.39100 22.64100 -H 1.13100 20.96100 23.23100 -O 23.21100 2.31100 26.03100 -H 22.50100 2.04100 26.68100 -H 22.95100 2.03100 25.10100 -O 27.28100 23.16100 25.04100 -H 26.96100 23.88100 24.42100 -H 27.52100 22.35100 24.51100 -O 0.96100 19.01100 26.15100 -H 0.24100 19.06100 25.46100 -H 0.72100 18.32100 26.84100 -O 23.89100 21.18100 21.90100 -H 24.16100 20.59100 21.15100 -H 23.89100 22.13100 21.59100 -O 24.28100 23.99100 27.27100 -H 24.40100 24.65100 26.53100 -H 24.74100 24.33100 28.10100 -O 21.69100 19.25100 24.80100 -H 21.58100 20.19100 25.13100 -H 21.64100 18.62100 25.57100 -O 1.02100 26.28100 0.45100 -H 1.17100 26.49100 1.41100 -H 0.72100 25.33100 0.36100 -O 26.77100 24.34100 21.87100 -H 26.84100 23.45100 21.41100 -H 25.83100 24.68100 21.79100 -O 23.35100 23.62100 20.53100 -H 23.96100 24.42100 20.57100 -H 22.40100 23.93100 20.60100 -O 21.25100 21.88100 25.82100 -H 20.46100 22.39100 25.48100 -H 21.16100 21.73100 26.80100 -O 27.78100 27.72100 21.53100 -H 28.41100 28.10100 20.85100 -H 28.18100 26.89100 21.92100 -O 1.18100 1.77100 25.58100 -H 0.48100 1.30100 26.12100 -H 1.77100 1.09100 25.15100 -O 19.37100 22.07100 18.95100 -H 18.45100 21.79100 18.66100 -H 19.68100 22.84100 18.39100 -O 19.34100 20.28100 21.80100 -H 19.17100 21.11100 21.26100 -H 20.24100 19.91100 21.58100 -O 24.56100 26.07100 25.14100 -H 25.06100 26.92100 24.95100 -H 23.68100 26.09100 24.66100 -O 1.62100 0.63100 19.49100 -H 1.39100 1.16100 18.67100 -H 2.43100 0.07100 19.30100 -O 22.90100 22.86100 23.82100 -H 23.20100 22.14100 23.20100 -H 22.51100 22.46100 24.65100 diff --git a/examples/gpmdk/run/coords_300.dat b/examples/gpmdk/run/coords_300.dat deleted file mode 100644 index 6eec1584..00000000 --- a/examples/gpmdk/run/coords_300.dat +++ /dev/null @@ -1,304 +0,0 @@ - 300 - 14.40500 0.00000 0.00000 - 0.00000 14.40500 0.00000 - 0.00000 0.00000 14.40500 -H 2.44100 0.20600 1.00300 -O 2.28300 0.06800 0.04900 -H 1.79100 13.63200 0.01400 -H 0.50700 14.25000 6.56400 -O 0.89500 0.74100 6.43800 -H 0.12600 1.30400 6.65900 -H 6.07100 4.56800 13.85400 -O 6.37600 5.36600 14.36300 -H 6.85400 4.94600 0.72800 -H 9.13800 4.82900 5.13900 -O 9.29500 4.91700 4.17100 -H 10.26100 4.85100 4.00400 -H 8.42200 9.67800 11.47900 -O 7.63900 10.22400 11.24900 -H 6.98500 10.03000 11.95900 -H 6.55500 7.29100 3.44600 -O 7.45600 7.68200 3.53200 -H 7.49900 7.84300 4.50800 -H 4.03200 0.78800 4.74000 -O 4.75200 0.59500 5.39200 -H 4.93100 14.05400 5.27900 -H 1.04800 12.26800 2.34300 -O 1.88300 12.29500 1.83400 -H 1.86500 11.48000 1.30400 -H 2.74800 5.73400 0.76500 -O 3.46700 5.76700 0.11100 -H 3.92400 4.91000 0.17200 -H 14.40100 8.01500 4.04100 -O 0.76200 8.54100 3.68700 -H 1.00500 8.03800 2.88300 -H 7.57100 5.54500 7.83500 -O 7.65800 5.22700 6.90000 -H 7.10000 4.43000 6.98000 -H 12.14000 7.26200 7.79000 -O 13.06300 7.56900 7.96500 -H 13.60300 6.79600 7.66600 -H 5.93400 8.72200 10.97800 -O 5.13500 8.66700 11.53300 -H 4.54900 8.03000 11.07400 -H 7.15000 0.49500 4.78800 -O 8.03300 0.64100 5.18500 -H 8.68400 0.54600 4.46200 -H 1.35800 11.09600 7.49700 -O 1.66800 11.00200 6.56900 -H 1.84200 10.05600 6.42900 -H 2.32300 0.33300 7.72900 -O 3.22700 14.38400 7.87200 -H 3.04600 13.58400 8.41900 -H 9.01000 4.35900 10.22700 -O 9.93700 4.62600 10.06700 -H 10.04200 4.45800 9.11500 -H 2.06900 2.69100 13.04500 -O 2.59600 2.79900 13.86800 -H 2.55700 1.89200 14.23700 -H 5.72700 4.79800 4.39600 -O 6.51700 5.06700 3.85800 -H 7.25200 4.96500 4.49800 -H 2.69800 5.51700 7.46500 -O 3.02200 6.39100 7.77500 -H 3.46900 6.21700 8.63900 -H 0.76700 4.55300 4.59100 -O 0.38900 3.83000 4.05200 -H 14.34500 4.32900 3.33800 -H 12.18900 4.33400 10.12700 -O 12.65400 4.89700 10.77400 -H 12.54400 4.41900 11.61900 -H 8.10500 13.66900 2.56500 -O 8.43200 13.12200 3.29900 -H 7.81600 12.42300 3.59800 -H 13.16500 9.90100 12.67200 -O 14.15900 9.90100 12.39300 -H 14.27000 9.05600 11.92800 -H 7.84000 13.93900 9.33300 -O 7.92300 14.23400 10.25800 -H 7.40000 13.58600 10.76200 -H 6.28400 0.41000 8.01800 -O 5.69000 1.17500 8.11800 -H 4.84100 0.82000 7.71700 -H 1.94900 12.85000 10.69600 -O 2.01500 12.54100 9.76500 -H 1.28400 11.89400 9.72200 -H 14.31000 7.21700 10.10700 -O 0.47300 7.78600 10.64900 -H 1.34500 7.36400 10.58100 -H 1.91600 0.79200 3.31800 -O 2.78500 0.62900 2.89000 -H 3.06600 14.17500 3.29700 -H 10.53100 2.50700 2.36700 -O 11.36600 2.91800 2.69700 -H 11.56000 2.47700 3.54000 -H 3.27600 5.45800 11.53400 -O 4.15400 5.85000 11.33800 -H 4.43800 6.12500 12.23200 -H 7.00100 8.93100 9.33700 -O 7.39500 8.40400 8.62600 -H 7.02100 7.49300 8.72600 -H 5.75200 5.35800 9.37400 -O 6.56800 5.90700 9.32100 -H 6.88200 5.97000 10.27200 -H 10.40400 12.64500 6.10700 -O 10.85000 13.13700 5.38900 -H 10.99800 12.46100 4.68700 -H 10.61900 10.58000 13.77000 -O 11.59400 10.52100 13.96100 -H 11.70300 9.55700 14.13200 -H 4.10700 1.50300 1.76500 -O 4.94200 1.40000 1.27600 -H 4.85000 0.48400 0.94400 -H 7.95300 12.37300 7.74500 -O 8.51700 12.17400 8.51900 -H 8.04200 11.47200 8.99800 -H 8.65200 3.68700 0.28100 -O 9.36200 3.39000 14.09900 -H 8.93600 2.79700 13.48000 -H 0.50300 8.64900 13.75200 -O 1.28100 8.48500 14.34500 -H 1.02900 7.68600 0.48100 -H 4.43000 12.76400 9.75100 -O 4.82500 13.52700 10.22600 -H 5.21400 14.07100 9.51700 -H 2.58000 9.53500 9.06300 -O 3.17400 8.77600 8.89400 -H 3.46100 8.85100 7.96600 -H 4.11900 12.68100 13.25000 -O 4.22500 12.06400 12.48900 -H 3.84800 11.22300 12.85000 -H 3.71200 13.00100 0.99200 -O 4.62800 12.95800 0.64700 -H 4.98900 12.13800 1.10600 -H 7.56200 9.50400 3.04100 -O 8.04500 10.34400 3.25300 -H 9.00100 10.12000 3.12500 -H 6.65100 11.71200 2.12800 -O 5.77600 11.41900 2.45900 -H 5.84800 10.44500 2.46400 -H 12.43900 6.66900 1.83900 -O 11.90900 6.01200 2.32300 -H 12.22900 5.14600 2.00300 -H 0.01500 5.86600 1.71600 -O 0.86300 6.29800 1.95400 -H 1.47200 5.58300 2.19800 -H 11.93100 12.50200 8.47200 -O 11.09900 12.76900 8.03300 -H 10.40400 12.33000 8.55900 -H 4.18500 7.37800 6.80400 -O 4.58800 8.16400 6.36200 -H 4.06900 8.26300 5.54600 -H 13.90600 2.01700 8.61300 -O 0.42000 2.40200 8.55700 -H 0.72200 2.32600 9.49200 -H 4.77800 3.30700 11.98200 -O 5.22100 2.98700 12.79900 -H 4.50400 3.06700 13.48000 -H 13.34400 13.91800 3.35700 -O 14.12600 14.28700 3.81400 -H 0.32500 13.51800 3.91800 -H 9.03700 6.80200 13.69500 -O 9.60000 7.46700 13.23300 -H 9.42900 7.30100 12.27400 -H 0.50800 11.30700 5.18600 -O 0.38100 11.38200 4.20800 -H 0.87200 10.59300 3.84800 -H 12.97800 3.42400 1.81800 -O 13.70400 3.93900 1.38500 -H 13.84500 3.49900 0.52700 -H 12.19600 6.94000 4.04700 -O 13.00400 7.21100 4.53000 -H 12.67700 7.39900 5.42900 -H 10.42200 0.27300 11.89100 -O 10.91400 0.38200 11.04600 -H 10.21900 0.31100 10.36600 -H 7.67500 2.21900 8.04800 -O 8.31300 2.58900 7.40300 -H 8.25800 1.94000 6.64500 -H 10.07400 10.19500 6.81700 -O 9.89800 10.16400 5.86100 -H 10.36600 9.35900 5.57300 -H 12.13400 1.07200 9.93600 -O 12.23300 1.27900 8.95600 -H 12.27200 0.42600 8.41600 -H 4.05900 7.71300 1.79200 -O 4.71500 7.10600 2.16000 -H 4.64800 6.38100 1.50800 -H 13.87700 10.61100 8.87100 -O 0.38900 10.36300 9.16200 -H 0.24000 9.46200 9.49100 -H 2.96700 9.01700 14.01200 -O 3.78500 9.58100 13.91400 -H 4.27700 9.04900 13.24700 -H 10.08500 14.00000 2.85900 -O 10.87300 14.37900 2.55900 -H 11.33800 0.39000 3.32600 -H 11.30900 7.53200 14.04400 -O 12.22900 7.70000 14.38900 -H 12.79700 7.04500 13.91200 -H 6.46100 13.93100 0.93500 -O 7.39600 13.67800 0.82500 -H 7.40600 13.36300 14.29800 -H 12.03000 1.44800 12.36400 -O 12.79400 1.37100 12.95000 -H 12.48500 0.71800 13.59900 -H 2.12200 3.25800 7.72200 -O 2.41000 3.77400 6.94200 -H 1.76400 3.47600 6.27600 -H 4.22800 3.98700 6.00400 -O 5.09600 3.72200 5.62800 -H 4.96000 2.80100 5.32800 -H 6.04700 9.52000 6.08700 -O 6.21900 10.28600 5.51600 -H 7.18900 10.39800 5.52700 -H 8.77800 12.11800 0.33800 -O 8.80000 11.14300 0.34500 -H 8.72500 10.88600 1.28400 -H 10.10700 3.02100 7.58500 -O 10.98000 3.50500 7.61600 -H 11.55900 2.82600 8.08000 -H 3.81500 0.37900 11.05900 -O 3.29900 1.19700 10.81100 -H 3.13700 1.04300 9.85600 -H 13.45000 5.79100 12.68600 -O 14.08500 6.44400 13.02200 -H 0.43100 6.35200 12.39500 -H 3.42700 12.06000 4.47100 -O 3.08300 12.95500 4.64600 -H 2.46000 12.84000 5.38600 -H 13.38900 10.00500 5.93700 -O 13.06200 10.09100 6.85900 -H 13.13100 9.17900 7.24100 -H 9.11400 7.98200 9.50300 -O 9.51600 7.58100 10.31200 -H 9.61800 6.62100 10.11700 -H 4.86000 3.20900 9.65100 -O 4.91300 3.71100 8.81700 -H 5.20200 3.03100 8.17800 -H 11.40400 9.36900 2.31000 -O 10.88500 9.86400 2.97000 -H 11.37800 9.78200 3.81300 -H 7.23000 3.78400 2.58200 -O 7.22100 3.20900 1.77800 -H 6.52200 2.53200 1.91800 -H 9.77400 8.20900 3.61000 -O 10.15200 7.57100 4.25100 -H 9.45400 6.89400 4.34800 -H 12.73400 13.13300 6.28600 -O 13.58500 13.10700 6.77900 -H 13.71100 12.13200 6.92500 -H 10.94300 4.25600 13.77000 -O 11.64400 4.91500 13.57600 -H 11.18700 5.75300 13.76000 -H 13.17500 4.66400 6.75600 -O 13.93700 4.92600 7.30800 -H 0.00800 4.08900 7.49900 -H 9.23200 0.02700 14.11100 -O 10.09800 14.15000 13.77100 -H 10.64300 14.14700 0.17500 -H 11.54100 0.53900 5.36700 -O 12.05300 1.30000 4.99700 -H 12.97700 0.98000 4.91300 -H 10.54600 11.50500 11.84100 -O 11.22400 11.83900 11.21900 -H 11.17800 12.81700 11.31600 -H 9.91800 6.03300 6.35500 -O 10.79300 6.15800 6.76000 -H 10.87500 5.36000 7.33500 -H 6.86300 2.48500 10.77100 -O 7.80800 2.52200 11.06600 -H 8.10000 1.59000 10.91800 -H 5.25000 12.79800 6.97500 -O 6.22400 12.78300 6.96300 -H 6.45400 12.10200 6.30800 -H 1.87300 3.69600 2.82600 -O 2.60200 3.96600 2.23300 -H 3.06200 3.14900 1.99600 -H 12.48600 12.31900 0.01700 -O 12.77300 13.11700 0.52100 -H 11.96200 13.45700 0.95200 -H 6.19400 12.38600 12.57000 -O 7.09800 12.71300 12.49900 -H 7.62200 12.00600 12.02800 -H 0.51200 1.65100 12.21400 -O 0.95800 2.07500 11.45200 -H 1.85400 1.62600 11.33700 -H 0.03200 13.04100 12.91800 -O 0.43200 12.61800 12.10900 -H 0.18700 11.64900 12.11500 -H 9.77600 5.76100 1.70900 -O 8.80000 5.83300 1.60300 -H 8.52300 6.45400 2.32500 -H 7.31400 5.72700 12.84400 -O 7.63500 5.59900 11.90600 -H 7.84700 4.59300 11.87700 -H 5.53800 9.45400 0.42400 -O 6.40700 9.29200 0.86200 -H 6.27900 8.38500 1.19900 -H 4.40000 10.34500 9.33100 -O 4.49200 11.09000 8.70400 -H 4.12200 10.74300 7.87400 -H 0.08700 4.80700 10.70500 -O 0.94000 4.68900 11.19200 -H 0.90200 3.73400 11.48000 diff --git a/examples/gpmdk/run/get_energy.py b/examples/gpmdk/run/get_energy.py deleted file mode 100755 index 2da423a3..00000000 --- a/examples/gpmdk/run/get_energy.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import re - -energy_re = re.compile("Energy Total.*=\s+([0-9-.]+)") - -parser = argparse.ArgumentParser() -parser.add_argument("OUT", - help="The output") -options = parser.parse_args() - -with open(options.OUT) as fd: - for line in fd: - result = energy_re.search(line) - if result: - print(result.group(1)) diff --git a/examples/gpmdk/run/gpmd_input.in b/examples/gpmdk/run/gpmd_input.in deleted file mode 100644 index 2ff3af1d..00000000 --- a/examples/gpmdk/run/gpmd_input.in +++ /dev/null @@ -1,153 +0,0 @@ -INPUT FILE FOR THE GPMD PROGRAM -=============================== - -#LATTE parameters -Latte{ - JobName= GPMD - #BMLType= Ellpack - BMLType= Dense - #Method= GSP2 - #Method= SP2 - #Method= Diag - #Method= DiagEf - Method= DiagEfFull - MDim= -1 - #Threshold= 1.0d-5 - Threshold= 0.0 - Verbose= 10 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) - #SCF variables# - StopAt= "gpmdcov_Energ" - #StopAt= "gpmdcov_DM_Min" - #StopAt= "gpmdcov_FirstCharges" - CoordsFile= "CH4.pdb" - MPulay= 10 - #ZMat= ZSP - ZMat= Diag - PulayCoeff= 0.1 - #MixCoeff= 0.6 #VALID FOR WAT - MixCoeff= 0.2 - SCFTol= 1.0d-8 - MaxSCFIter= 500 - CoulAcc= 1.0d-5 - TimeRatio= 10.0 - TimeStep= 0.1 - #TimeStep= 0.00 - MDSteps= 200000 - ParamPath= "../sulfurTBparam" - NlistEach= 1 - MuCalcType= FromParts - EFermi= -0.0 - kBT= 0.025 - Entropy= T - DoKernel= F -} - -#SP2 Solver -SP2{ - MinSP2Iter= 10 - MaxSP2Iter= 200 - SP2Tol= 1.0d-5 - SP2Conv= Rel -} - -#Graph-based SP2 parameters -GSP2{ - - BMLType= Ellpack - #GraphElement= Orbital - GraphElement= Atom - #GraphElement= Orbital - #PartitionType= Block - #NodesPerPart= 333 - #NodesPerPart= 18 - #NodesPerPart= 27 - #NodesPerPart= 512 - #NodesPerPart= 17 - #NodesPerPart= 48 - #NodesPerPart= 150 - #NodesPerPart= 1331 - #PartitionType= METIS - PartitionType= Block - NodesPerPart= 0 - #PartitionType= METIS+SA - #PartitionType= METIS+KL - #PartitionRefinement= None - #PartitionRefinement= KL - #PartitionRefinement= SA - NLGraphCut= 4.5 - CovGraphFact= 4.5 - #CovGraphFact= 3.0 - #PartitionCount= 128 - #PartitionCount= 4 - PartitionCount= 1 - #PartitionCount= 512 - #PartitionCount= 16 - #PartitionCount= 2 - #PartitionCount= 8 - #PartitionCount= 1024 - #PartitionCount= 32 - #PartitionCount= 64 - #PartitionCount= 3200 - #PartitionCount= 6400 - #PartitionCount= 1 - GraphThreshold= 0.005 - ErrLimit= 1.0e-12 - PartEach= 100 - Mdim= -1 -} - - -#Sparse propagation of the inverse overlap -ZSP{ - Verbose= 1 - NFirst= 8 - NRefI= 3 - NRefF= 1 - Int= .true. - NumthreshI= 1.0d-8 - NumthreshF= 1.0d-5 -} - -#Extended Lagrangian parameters -XLBO{ - JobName= XLBO - Verbose= 1 - Mprg_init= 2 - MaxSCFIter= 0 - MaxSCFInitIter= 50 - NumThresh= 0.0 -} - - -KERNEL{ - KernelType= ByParts - BuildAlways= F - RankNUpdate= 3 - KernelMixing= F - InitialMixingWith= DIIS - UpdateEach= 1 - UpdateAfterBuild= T - Verbose= 1 -} - -GPMD{ - DoVelocityRescale= F - #VRFactor= 1.0 - WriteTrajectory= T - WriteCoordsEach= 1 - #TrajectoryFormat= XYZ - InitialTemperature= 300 - #HtoD= T - MinimizationSteps= 100 - LangevinDynamics= T - LangevinMethod= Siva - UseSMD= F - #For polyaniline - #SMDAtomIndex1= 6 - #SMDAtomIndex2= 7 - - SMDForceConstant= -8.0 - SMDR0= 0.001 - ComputeCurrents= T - CurrentThreshold= 0.0 -} diff --git a/examples/gpmdk/run/input.in b/examples/gpmdk/run/input.in deleted file mode 100644 index e85741ee..00000000 --- a/examples/gpmdk/run/input.in +++ /dev/null @@ -1,182 +0,0 @@ -INPUT FILE FOR THE GPMD PROGRAM -=============================== - -#LATTE parameters -Latte{ - JobName= GPMD - #BMLType= Ellpack - BMLType= Dense - #Method= GSP2 - #Method= SP2 - #Method= Diag - #Method= DiagEf - Method= DiagEfFull - MDim= -1 - #Threshold= 1.0d-5 - Threshold= 0.0 - Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) - #SCF variables# - #StopAt= "gpmdcov_Energ" - #StopAt= "gpmdcov_DM_Min" - #StopAt= "gpmdcov_FirstCharges" - MPulay= 10 - #ZMat= ZSP - ZMat= Diag - PulayCoeff= 0.1 - #MixCoeff= 0.6 #VALID FOR WAT - MixCoeff= 0.2 - SCFTol= 1.0d-8 - MaxSCFIter= 500 - CoulAcc= 1.0d-5 - TimeRatio= 10.0 - TimeStep= 0.2 - #TimeStep= 0.00 - MDSteps= 10000 - #ParamPath= "../sulfurTBparam" - ParamPath= "../tests/latteTBparams" - #ParamPath= "../latteTBparams_orig" - #CoordsFile= coords.ltt - #CoordsFile= coords_300New.dat - #CoordsFile= coords_300_sort.dat - #CoordsFile= coords_300.dat - CoordsFile= coords_2088.dat - #CoordsFile= "./polyaniline.pdb" - #CoordsFile= graphite2048.pdb - #CoordsFile= carbon_2197.pdb - #CoordsFile= ammonia2889.pdb - #CoordsFile= amonia1583.pdb - #CoordsFile= ammonia9863.pdb - #CoordsFile= ammonia4071.pdb - #CoordsFile= ammonia6610.pdb - #CoordsFile= Tr_Cage_opt.pdb - #CoordsFile= Tr_Cage.pdb - #CoordsFile= water3.pdb - #CoordsFile= carbon.pdb - #CoordsFile= carbon343.pdb - #CoordsFile= O2_solv114.pdb - #CoordsFile= full_tol360.pdb - #CoordsFile= full.pdb - #CoordsFile= NCH4.dat - #CoordsFile= full_tol3240.pdb - #CoordsFile= full_tol1440.pdb - #CoordsFile= water_box.pdb - #CoordsFile= coords_100WatAnders.dat - #CoordsFile= rand_H2O_1000.dat - #CoordsFile= coords_1032.dat - #CoordsFile= coords_2088.dat - #CoordsFile= coords_2955.pdb - #CoordsFile= amonia333.pdb - #CoordsFile= ammonia85.pdb - #CoordsFile= water3.dat - #CoordsFile= coords_10000.pdb - #CoordsFile= coords_6495.pdb - #CoordsFile= coords_30000.pdb - #CoordsFile= coords_65000.pdb - #CoordsFile= coords_100000.pdb - NlistEach= 1 - #BMLDistributionType= Distributed - MuCalcType= FromParts - #MuCalcType= Combined - #MuCalcType= Dyn - EFermi= -0.0 - #kBT= 0.04308695 - kBT= 0.025 - Entropy= T - DoKernel= T -} - -#SP2 Solver -SP2{ - MinSP2Iter= 10 - MaxSP2Iter= 200 - SP2Tol= 1.0d-5 - SP2Conv= Rel -} - -#Graph-based SP2 parameters -GSP2{ - - BMLType= Ellpack - #GraphElement= Orbital - GraphElement= Atom - #PartitionType= Block - #NodesPerPart= 333 - #NodesPerPart= 18 - #NodesPerPart= 27 - #NodesPerPart= 512 - #NodesPerPart= 17 - #NodesPerPart= 48 - #NodesPerPart= 150 - #NodesPerPart= 1331 - PartitionType= METIS - #PartitionType= METIS+SA - #PartitionType= METIS+KL - #PartitionRefinement= None - #PartitionRefinement= KL - #PartitionRefinement= SA - NLGraphCut= 4.5 - CovGraphFact= 4.5 - #CovGraphFact= 3.0 - #PartitionCount= 128 - #PartitionCount= 256 - #PartitionCount= 512 - #PartitionCount= 16 - #PartitionCount= 1 - PartitionCount= 8 - #PartitionCount= 1024 - #PartitionCount= 32 - #PartitionCount= 3200 - #PartitionCount= 6400 - #PartitionCount= 1 - GraphThreshold= 0.005 - ErrLimit= 1.0e-12 - PartEach= 100 - Mdim= -1 -} - - -#Sparse propagation of the inverse overlap -ZSP{ - Verbose= 1 - NFirst= 8 - NRefI= 3 - NRefF= 1 - Int= .true. - NumthreshI= 1.0d-8 - NumthreshF= 1.0d-5 -} - -#Extended Lagrangian parameters -XLBO{ - JobName= XLBO - Verbose= 1 - Mprg_init= 2 - MaxSCFIter= 0 - MaxSCFInitIter= 50 - NumThresh= 0.0 -} - - -KERNEL{ - KernelType= ByParts - #KernelType= Full - #KernelType= ByBlocks - BuildAlways= F - RankNUpdate= 4 - KernelMixing= T - InitialMixingWith= DIIS - UpdateEach= 1 - UpdateAfterBuild= T - Verbose= 1 -} - -GPMD{ - DoVelocityRescale= F - #VRFactor= 1.0 - WriteTrajectory= T - WriteCoordsEach= 10 - LangevinMethod= Siva - LangevinDynamics= T - LangevinGamma= 0.01 - InitialTemperature= 300.0 -} diff --git a/examples/gpmdk/run/job b/examples/gpmdk/run/job deleted file mode 100755 index 1ecc0cf0..00000000 --- a/examples/gpmdk/run/job +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -#SBATCH -N32 #Number of cores and nodes -##SBATCH -N1 #Number of cores and nodes -#SBATCH --ntasks-per-node=72 -#SBATCH -p scaling --time=8:00:00 -# OMP_NUM_THREADS=36 controls the number of threads your application use -## This variable cat be set by the following command : - -source $HOME/BESGraph/qmd-progress/examples/gpmdcov/vars - -## Run command -export OMP_NUM_THREADS=36 -RUN="$HOME/BESGraph/qmd-progress/build/gpmdcov" -#OMP_PROC_BIND=spread OMP_NUM_THREADS=4 mpirun -n 128 --map-by ppr:8:node:pe=4 $RUN input.in > out -#OMP_PROC_BIND=spread OMP_NUM_THREADS=36 mpirun -n 1 --map-by ppr:4:node:pe=36 $RUN input.in > out -#OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 64 --map-by ppr:4:node:pe=9 $RUN input.in > out -OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 16 --map-by ppr:4:node:pe=9 $RUN input.in > out - diff --git a/examples/gpmdk/run/latte.in b/examples/gpmdk/run/latte.in deleted file mode 100644 index 59947cd5..00000000 --- a/examples/gpmdk/run/latte.in +++ /dev/null @@ -1,74 +0,0 @@ -LATTE INPUT FILE -================ -#This input file resumes the content of MDcontroller and TBparam/control.in -#The parser will only read it if it is present inside the running folder. -#In case this file is not present Latte will read the two files as original. -#The order of the kewords is not important in this file. - -#General controls -CONTROL{ - XCONTROL= 1 - BASISTYPE= NONORTHO - PARAMPATH= "./latteTBparams" - DEBUGON= 0 - FERMIM= 6 - CGORLIB= 1 CGTOL= 1.0e-6 - KBT= 0.0 - NORECS= 1 - ENTROPYKIND= 1 - PPOTON= 1 VDWON= 0 - SPINON= 0 SPINTOL= 1.0e-4 - ELECTRO= 0 ELECMETH= 1 ELEC_ETOL= 0.001 ELEC_QTOL= 1.0e-4 - COULACC= 1.0e-6 COULCUT= -500.0 COULR1= 500.0 - MAXSCF= 250 - BREAKTOL= 1.0E-12 MINSP2ITER= 22 SP2CONV= REL - FULLQCONV= 1 QITER= 0 - QMIX= 0.25 SPINMIX= 0.25 MDMIX= 0.25 - ORDERNMOL= 0 - SPARSEON= 0 THRESHOLDON= 1 NUMTHRESH= 1.0e-6 FILLINSTOP= 100 BLKSZ= 4 - MSPARSE= 3000 - LCNON= 0 LCNITER= 4 CHTOL= 0.01 - SKIN= 1.0 - RELAX= 0 RELAXTYPE= SD MAXITER= 100 RLXFTOL= 0.00001 - MDON= 0 - PBCON= 1 - RESTART= 0 - CHARGE= 0 - XBO= 1 - XBODISON= 1 - XBODISORDER= 5 - NGPU= 2 - KON= 0 - COMPFORCE= 1 - DOSFIT= 0 INTS2FIT= 1 BETA= 1000.0 NFITSTEP= 5000 QFIT= 0 MCSIGMA= 0.2 - PPFITON= 0 - ALLFITON= 0 - PPSTEP= 500 BISTEP= 500 PP2FIT= 2 BINT2FIT= 6 - PPBETA= 1000.0 PPSIGMA= 0.01 PPNMOL= 10 PPNGEOM= 200 - PARREP= 0 - ER= 1.0 -} - -#Controls for QMD -MDCONTROL{ - MAXITER= 20 - UDNEIGH= 1 - DT= 0.25 - TEMPERATURE= 1.0e-30 RNDIST= GAUSSIAN SEEDINIT= UNIFORM - DUMPFREQ= 250 - RSFREQ= 500 - WRTFREQ= 1 - TOINITTEMP5= 1 - THERMPER= 500 - THERMRUN= 50000 - NVTON= 0 NPTON= 0 AVEPER= 1000 FRICTION= 1000.0 SEED= 54 - PTARGET= 0.0 NPTTYPE= ISO - SHOCKON= 0 - SHOCKSTART= 100000 - SHOCKDIR= 1 - UPARTICLE= 500.0 USHOCK= -4590.0 C0= 1300.0 - MDADAPT= 0 - GETHUG= 0 E0= -795.725 V0= 896.984864 P0= 0.083149 -} - - diff --git a/examples/gpmdk/run/run.sh b/examples/gpmdk/run/run.sh deleted file mode 100755 index 0696145d..00000000 --- a/examples/gpmdk/run/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - - -#RUN="$PWD/../build_hackathon/gpmdk" -#OMP_NUM_THREADS=32 mpiexec -np 3 $RUN input.in | tee out -#OMP_NUM_THREADS=36 mpirun -np 1 $RUN input.in | tee out -#OMP_NUM_THREADS=36 mpiexec -np 1 $RUN input.in | tee out - -#OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out -OMP_NUM_THREADS=32 srun -n 4 -c 32 bash wrapper.sh -#OMP_NUM_THREADS=64 $RUN input.in | tee out -#OMP_NUM_THREADS=36 $RUN input.in | tee out -#$RUN input.in | tee out - diff --git a/examples/gpmdk/run/sbatch.sh b/examples/gpmdk/run/sbatch.sh deleted file mode 100755 index 27a1b9e7..00000000 --- a/examples/gpmdk/run/sbatch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -##SBATCH -p gpu -#SBATCH -p gpu_debug -#SBATCH --reservation=gpu_debug -#SBATCH --qos=debug -##SBATCH --time 16:00:00 -#SBATCH --time 2:00:00 -#SBATCH -N 2 -##SBATCH -A ichelp_g -#SBATCH -A w23_macroqmd_g - -cd ${PWD} - -source /usr/projects/icapt/mewall/packages/gpmd/gpmd/setenv_gpu.sh - -#export MPICH_ALLREDUCE_NO_SMP=1 -export MPICH_SMP_SINGLE_COPY_MODE=NONE - -OMP_NUM_THREADS=32 srun -n 8 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh - diff --git a/examples/gpmdk/run/wrapper.sh b/examples/gpmdk/run/wrapper.sh deleted file mode 100644 index 2a0f668e..00000000 --- a/examples/gpmdk/run/wrapper.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) - -RUN="$PWD/../build_hackathon/gpmdk" -#stdbuf -o0 $RUN input_ICH.in |& tee out_ICH -if [ $SLURM_PROCID -eq 0 ]; then - $RUN input.in >& out -else - $RUN input.in >& /dev/null -fi diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index ee8b0579..2209d241 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -42,8 +42,9 @@ subroutine gpmdcov_Part(ipreMD) if(gpat%TotalParts > 1)then call gpmdcov_msI("gpmdcov_Part","In prg_get_covgraph_h ...",lt%verbose,myRank) mls_ii = mls() +#ifdef DO_MPI call prg_get_covgraph_h(sy,nl%nnStruct,nl%nrnnstruct,gsp2%nlgcut,graph_h,myMdim,lt%verbose) - call gpmdcov_msIII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) #ifdef DO_MPI !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) @@ -66,6 +67,7 @@ subroutine gpmdcov_Part(ipreMD) #ifdef DO_MPI if (getNRanks() > 1) then + call prg_barrierParallel call prg_sumIntReduceN(graph_p, myMdim*sy%nats) ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) endif @@ -74,14 +76,14 @@ subroutine gpmdcov_Part(ipreMD) ! call gpmdcov_vect2MatInt(auxVectInt,graph_p,sy%nats,myMdim) ! deallocate(auxVectInt) ! write(*,*)graph_p - call gpmdcov_msIII("gpmdcov_Part","Time for prg_sumIntReduceN for graph "//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Part","Time for prg_sumIntReduceN for graph "//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) call gpmdcov_msI("gpmdcov_Part","In prg_merge_graph ...",lt%verbose,myRank) mls_ii = mls() call prg_wait() call prg_merge_graph(graph_p,graph_h) call prg_wait() - call gpmdcov_msIII("gpmdcov_Part","Time for prg_merge_graph "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Part","Time for prg_merge_graph "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) !call prg_wait() !call prg_wait() @@ -107,7 +109,7 @@ subroutine gpmdcov_Part(ipreMD) call gpmdcov_msMem("gpmdcov_Part","After prg_graph2bml",lt%verbose,myRank) - call gpmdcov_msIII("gpmdcov_Part","Time for prg_graph2bml "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Part","Time for prg_graph2bml "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) !endif else allocate(onesMat(sy%nats,sy%nats)) @@ -216,7 +218,7 @@ subroutine gpmdcov_Part(ipreMD) endif endif - call gpmdcov_msIII("gpmdcov_Part","Time for bml_matrix2submatrix_index "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + call gpmdcov_msII("gpmdcov_Part","Time for bml_matrix2submatrix_index "//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) call gpmdcov_reshuffle() From ff1bd08095572530b672279ee6c353d26c8b88e6 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 5 Jul 2024 11:47:33 -0600 Subject: [PATCH 07/69] Add venado build, env, and run scripts --- build_venado_hackathon.sh | 43 + examples/gpmdk/build_venado_hackathon.sh | 10 + examples/gpmdk/run/water/coords_2088.dat | 2092 ++++++++++++++++++++++ examples/gpmdk/run/water/coords_300.dat | 304 ++++ examples/gpmdk/run/water/get_energy.py | 17 + examples/gpmdk/run/water/gpmd_input.in | 153 ++ examples/gpmdk/run/water/input.in | 182 ++ examples/gpmdk/run/water/input_single.in | 182 ++ examples/gpmdk/run/water/job | 19 + examples/gpmdk/run/water/latte.in | 74 + examples/gpmdk/run/water/run.sh | 14 + examples/gpmdk/run/water/run_single.sh | 14 + examples/gpmdk/run/water/sbatch.sh | 21 + examples/gpmdk/run/water/wrapper.sh | 10 + scripts/setenv_venado.sh | 3 + 15 files changed, 3138 insertions(+) create mode 100755 build_venado_hackathon.sh create mode 100755 examples/gpmdk/build_venado_hackathon.sh create mode 100644 examples/gpmdk/run/water/coords_2088.dat create mode 100644 examples/gpmdk/run/water/coords_300.dat create mode 100755 examples/gpmdk/run/water/get_energy.py create mode 100644 examples/gpmdk/run/water/gpmd_input.in create mode 100644 examples/gpmdk/run/water/input.in create mode 100644 examples/gpmdk/run/water/input_single.in create mode 100755 examples/gpmdk/run/water/job create mode 100644 examples/gpmdk/run/water/latte.in create mode 100755 examples/gpmdk/run/water/run.sh create mode 100755 examples/gpmdk/run/water/run_single.sh create mode 100755 examples/gpmdk/run/water/sbatch.sh create mode 100644 examples/gpmdk/run/water/wrapper.sh create mode 100644 scripts/setenv_venado.sh diff --git a/build_venado_hackathon.sh b/build_venado_hackathon.sh new file mode 100755 index 00000000..697ef254 --- /dev/null +++ b/build_venado_hackathon.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Make a copy of this script and modify the METIS and BML library locations + +rm -r build +#rm -r install_magma_2.7.2 +rm -r install_hackathon + +# Set METIS and BML Library locations +METIS_LIB="/usr/projects/icapt/mewall/venado/packages/metis-5.1.0/install" +export BML_LIB=${BML_LIB:="/usr/projects/icapt/mewall/venado/packages/qmd-progress/bml/install_hackathon"} +#BML_LIB="/usr/projects/icapt/mewall/packages/gpmd/bml/install_magma_2.7.2" + +MY_PATH=`pwd` + +# Configuring PROGRESS with OpenMP +export CC=${CC:=cc} +export FC=${FC:=ftn} +export CXX=${CXX:=CC} +export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ffixed-line-length-none"} +#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"} +export PKG_CONFIG_PATH="$BML_LIB/lib/pkgconfig:$BML_LIB/lib64/pkgconfig:$PKG_CONFIG_PATH" +export PROGRESS_OPENMP=${PROGRESS_OPENMP:=yes} +export PROGRESS_MPI=${PROGRESS_MPI:=yes} +export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_hackathon"} +export PROGRESS_GRAPHLIB=${PROGRESS_GRAPHLIB:=yes} +export PROGRESS_TESTING=${PROGRESS_TESTING:=yes} +export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release} +export PROGRESS_EXAMPLES=${PROGRESS_EXAMPLES:=yes} +#export PROGRESS_BENCHMARKS=yes +export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=$METIS_LIB/:$BML_LIB/} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch -fdefault-integer-8"} +export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +export CMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH:="$METIS_LIB/include"} +export CMAKE_LIBRARY_PATH=${CMAKE_LIBRARY_PATH:="$METIS_LIB/lib"} +#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" +./build.sh configure + +# Make PROGRESS library and examples after running this script: +# cd build +# make diff --git a/examples/gpmdk/build_venado_hackathon.sh b/examples/gpmdk/build_venado_hackathon.sh new file mode 100755 index 00000000..465d4b24 --- /dev/null +++ b/examples/gpmdk/build_venado_hackathon.sh @@ -0,0 +1,10 @@ +WORKDIR=/usr/projects/icapt/mewall/venado/packages +rm -rf build_hackathon +mkdir build_hackathon +cd build_hackathon +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" \ +-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero" \ +-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ +#-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ +make + diff --git a/examples/gpmdk/run/water/coords_2088.dat b/examples/gpmdk/run/water/coords_2088.dat new file mode 100644 index 00000000..e0a6c820 --- /dev/null +++ b/examples/gpmdk/run/water/coords_2088.dat @@ -0,0 +1,2092 @@ + 2088 + 27.83000 0.00000 0.00000 + 0.00000 27.83000 0.00000 + 0.00000 0.00000 27.83000 + O 13.72700 13.73400 13.91500 + H 13.30900 14.59500 13.91500 + H 14.70800 13.41500 13.91500 +O 2.30000 6.28000 1.13000 +H 1.37000 6.26000 1.50000 +H 2.31000 5.89000 0.21000 +O 5.69000 12.75100 11.65100 +H 4.76000 12.68100 11.28100 +H 5.80000 13.64100 12.09100 +O 15.55100 15.11100 7.03000 +H 14.98100 14.95100 7.84000 +H 14.96100 15.21100 6.23000 +O 17.43100 6.18000 8.56000 +H 17.76100 7.12000 8.56000 +H 17.94100 5.64000 9.22000 +O 11.35100 7.03000 7.17000 +H 11.92100 7.81000 6.92000 +H 10.75100 7.29000 7.93000 +O 17.55100 6.07000 2.31000 +H 17.43100 5.94000 1.32000 +H 17.25100 5.26000 2.80000 +O 7.68000 11.44100 10.23100 +H 6.90000 11.61100 10.83100 +H 8.02000 12.31100 9.87100 +O 8.50000 7.98000 18.23100 +H 8.46000 8.74000 18.88100 +H 8.72000 8.34000 17.32100 +O 6.85000 10.12100 6.65000 +H 7.54000 9.96100 7.35000 +H 6.12000 10.69100 7.03000 +O 6.86000 11.61100 18.03100 +H 7.46000 12.40100 18.17100 +H 6.00000 11.92100 17.62100 +O 3.35000 14.35100 10.61100 +H 2.57000 14.04100 10.08100 +H 3.93000 14.93100 10.04100 +O 4.38000 3.92000 14.99100 +H 5.20000 3.36000 15.08100 +H 3.57000 3.34000 15.03100 +O 16.03100 4.47000 7.37000 +H 15.29100 4.93000 6.87000 +H 16.54100 5.15000 7.90000 +O 2.31000 17.13100 4.83000 +H 2.65000 17.90100 5.37000 +H 2.75000 17.13100 3.93000 +O 11.27100 13.41100 16.90100 +H 11.74100 13.41100 17.78100 +H 10.79100 12.54100 16.79100 +O 2.40000 10.91100 8.86000 +H 2.54000 10.07100 9.38000 +H 1.85000 11.55100 9.41000 +O 6.20000 17.86100 14.39100 +H 5.28000 17.69100 14.74100 +H 6.48000 18.78100 14.65100 +O 6.06000 9.64100 1.23000 +H 6.13000 10.48100 0.69000 +H 6.52000 9.77100 2.11000 +O 15.94100 1.14000 14.80100 +H 15.76100 1.81000 14.08100 +H 15.91100 1.60000 15.69100 +O 18.42100 17.67100 3.59000 +H 18.96100 17.38100 4.39000 +H 18.72100 18.57100 3.30000 +O 16.89100 9.22000 6.12000 +H 17.84100 8.93000 6.20000 +H 16.81100 9.87000 5.37000 +O 16.41100 11.08100 4.32000 +H 17.27100 11.10100 3.80000 +H 16.55100 11.55100 5.20000 +O 6.13000 13.65100 7.26000 +H 5.64000 12.78100 7.35000 +H 5.90000 14.08100 6.39000 +O 8.09000 0.04000 5.02000 +H 8.49000 0.95000 4.93000 +H 7.09000 0.12000 5.08000 +O 15.25100 9.99100 1.90000 +H 14.62100 9.23100 2.03000 +H 15.73100 10.17100 2.76000 +O 11.87100 17.92100 16.16100 +H 12.11100 18.52100 15.40100 +H 11.94100 16.97100 15.86100 +O 3.17000 2.51000 18.01100 +H 3.88000 3.22000 18.07100 +H 2.29000 2.90000 18.29100 +O 14.66100 14.17100 9.53100 +H 14.07100 14.23100 10.33100 +H 14.51100 13.29100 9.07100 +O 16.67100 17.14100 5.72000 +H 16.26100 16.91100 4.84000 +H 16.49100 16.40100 6.37000 +O 5.98000 7.29000 2.70000 +H 6.22000 7.98000 2.02000 +H 5.20000 7.62000 3.24000 +O 12.81100 3.45000 9.44100 +H 11.95100 2.95000 9.31100 +H 13.43100 2.91000 10.00100 +O 15.76100 16.62100 3.07000 +H 16.65100 17.08100 3.10000 +H 15.55100 16.38100 2.12000 +O 8.07000 6.05000 14.65100 +H 7.60000 6.02000 15.54100 +H 7.56000 5.50000 13.99100 +O 13.94100 4.69000 16.74100 +H 13.74100 5.12000 17.62100 +H 14.72100 4.07000 16.83100 +O 9.73100 8.90000 15.72100 +H 10.19100 8.06000 15.43100 +H 9.17100 9.24000 14.97100 +O 9.91100 4.10000 12.42100 +H 9.14100 4.44000 12.96100 +H 9.57100 3.59000 11.63100 +O 10.41100 7.01000 4.29000 +H 10.67100 6.97000 5.25000 +H 9.56100 6.50000 4.15000 +O 8.65000 3.48000 1.95000 +H 9.24000 4.11000 1.46000 +H 8.84000 2.54000 1.66000 +O 17.19100 5.85000 18.31100 +H 16.93100 6.74000 17.95100 +H 17.17100 5.17000 17.58100 +O 13.62100 11.44100 5.45000 +H 14.45100 11.15100 4.97000 +H 13.13100 12.11100 4.89000 +O 5.50000 1.96000 8.85000 +H 5.45000 1.91000 9.85000 +H 5.52000 2.92000 8.56000 +O 10.08100 14.56100 4.77000 +H 9.62100 15.28100 4.25000 +H 10.04100 14.76100 5.75000 +O 3.51000 18.01100 8.53000 +H 4.01000 17.15100 8.59000 +H 4.16000 18.78100 8.50000 +O 17.95100 10.66100 8.73000 +H 17.33100 10.51100 7.97000 +H 17.43100 10.77100 9.58000 +O 12.27100 15.50100 15.06100 +H 12.33100 14.73100 15.70100 +H 11.75100 15.24100 14.26100 +O 3.21000 9.43100 2.42000 +H 4.03000 9.82100 2.00000 +H 2.94000 8.61100 1.93000 +O 14.58100 7.35000 7.28000 +H 14.53100 6.70000 8.03000 +H 15.38100 7.94000 7.41000 +O 4.61000 12.66100 17.27100 +H 4.11000 12.67100 16.41100 +H 3.98000 12.48100 18.03100 +O 11.11100 17.76100 2.37000 +H 10.51100 17.14100 2.87000 +H 11.42100 17.32100 1.52000 +O 2.02000 2.85000 14.98100 +H 1.22000 3.45000 14.85100 +H 1.92000 2.36000 15.84100 +O 16.32100 13.77100 0.81000 +H 16.00100 14.71100 0.71000 +H 15.56100 13.14100 0.69000 +O 4.64000 17.43100 3.23000 +H 4.97000 17.82100 4.09000 +H 5.40000 17.36100 2.58000 +O 14.00100 1.07000 4.26000 +H 13.76100 0.70000 3.36000 +H 14.99100 1.23000 4.30000 +O 9.40100 16.98100 9.04000 +H 10.20100 16.41100 9.25000 +H 8.91100 16.58100 8.27000 +O 3.82000 7.00000 4.80000 +H 4.27000 6.10000 4.77000 +H 2.88000 6.89000 5.13000 +O 15.47100 2.22000 17.29100 +H 15.42100 2.59000 18.21100 +H 14.75100 1.53000 17.17100 +O 6.14000 1.22000 1.17000 +H 7.12000 1.00000 1.24000 +H 5.83000 1.05000 0.24000 +O 7.81000 2.64000 17.49100 +H 8.48000 2.03000 17.92100 +H 7.08000 2.83000 18.14100 +O 8.88000 15.14100 11.95100 +H 8.65000 14.89100 11.01100 +H 9.49000 14.45100 12.34100 +O 13.51100 5.90000 14.33100 +H 13.79100 5.47000 15.18100 +H 13.76100 6.86000 14.34100 +O 8.03000 14.02100 9.24000 +H 8.93000 14.16100 8.82000 +H 7.32000 14.04100 8.53000 +O 9.22000 5.03000 8.99000 +H 8.97000 4.94000 8.03000 +H 9.70000 4.21000 9.30000 +O 5.39000 0.64000 5.12000 +H 4.58000 0.65000 5.70000 +H 5.42000 1.47000 4.57000 +O 14.34100 11.88100 0.41000 +H 14.66100 11.12100 0.98000 +H 13.42100 12.15100 0.71000 +O 2.97000 0.35000 1.71000 +H 3.46000 1.19000 1.50000 +H 3.59000 -0.30000 2.16000 +O 9.35100 2.36000 4.80000 +H 8.87100 2.77000 4.02000 +H 10.34100 2.34000 4.61000 +O 10.76100 6.83000 14.64100 +H 9.96100 6.22000 14.67100 +H 11.57100 6.30000 14.40100 +O 12.27100 15.70100 7.93000 +H 12.48100 16.44100 7.28000 +H 12.95100 15.70100 8.66000 +O 4.59000 11.52100 7.41000 +H 3.88000 11.25100 8.06000 +H 4.33000 11.24100 6.48000 +O 12.71100 17.97100 5.91000 +H 13.15100 18.61100 5.27000 +H 12.21100 18.49100 6.61000 +O 10.32100 5.49000 0.16000 +H 9.91100 6.31000 -0.23000 +H 10.96100 5.75000 0.89000 +O 0.78000 5.56000 13.86100 +H 1.70000 5.55000 13.45100 +H 0.72000 6.30000 14.53100 +O 5.61000 2.22000 11.47100 +H 5.99000 1.38000 11.84100 +H 4.73000 2.41000 11.91100 +O 8.66000 4.54000 6.42000 +H 8.34000 5.26000 5.80000 +H 8.90000 3.73000 5.89000 +O 10.17100 0.39000 7.53000 +H 9.45100 0.44000 6.84000 +H 9.93100 -0.30000 8.22000 +O 14.29100 11.73100 8.67000 +H 13.74100 10.89100 8.60000 +H 14.55100 12.02100 7.75000 +O 14.66100 5.90000 9.92100 +H 14.36100 4.95000 9.99100 +H 15.39100 6.06000 10.58100 +O 18.57100 8.33000 3.77000 +H 18.99100 7.69000 4.41000 +H 18.19100 7.82000 2.99000 +O 4.88000 13.85100 1.74000 +H 4.01000 13.70100 2.21000 +H 4.71000 14.11100 0.79000 +O 16.64100 12.80100 6.57000 +H 17.63100 12.88100 6.71000 +H 16.19100 13.64100 6.88000 +O 13.90100 5.75000 0.78000 +H 13.36100 5.54000 1.59000 +H 14.81100 5.34000 0.87000 +O 5.27000 2.56000 3.28000 +H 5.54000 1.97000 2.53000 +H 5.27000 3.51000 2.97000 +O 17.54100 12.23100 15.88100 +H 18.45100 11.84100 15.75100 +H 17.62100 13.19100 16.12100 +O 10.64100 13.47100 13.40100 +H 9.84100 13.24100 13.95100 +H 11.47100 13.21100 13.89100 +O 15.92100 16.29100 16.25100 +H 16.19100 16.63100 15.35100 +H 16.71100 15.91100 16.71100 +O 11.11100 11.95100 11.00100 +H 10.71100 12.39100 11.81100 +H 10.70100 12.32100 10.17100 +O 16.38100 10.99100 10.79100 +H 16.43100 11.80100 11.38100 +H 15.52100 11.01100 10.28100 +O 9.15000 0.89000 14.02100 +H 9.40000 0.69000 13.07100 +H 9.87000 1.45000 14.44100 +O 9.80100 11.16100 17.19100 +H 8.81100 11.22100 17.29100 +H 10.03100 10.36100 16.63100 +O 7.05000 10.50100 3.68000 +H 6.91000 10.57100 4.67000 +H 7.89000 9.99100 3.50000 +O 4.10000 8.13000 12.51100 +H 4.96000 8.25000 13.01100 +H 3.68000 7.26000 12.78100 +O 12.74100 3.86000 12.62100 +H 12.95100 4.60000 13.26100 +H 11.85100 4.03000 12.19100 +O 3.67000 11.00100 5.01000 +H 3.60000 11.83100 4.45000 +H 3.71000 10.20100 4.41000 +O 5.66000 5.37000 8.65000 +H 5.78000 6.03000 7.91000 +H 6.12000 5.71000 9.48000 +O 12.52100 13.48100 3.88000 +H 13.02100 14.25100 4.28000 +H 11.57100 13.50100 4.20000 +O 12.72100 14.45100 11.42100 +H 13.19100 14.58100 12.29100 +H 12.06100 13.71100 11.51100 +O 15.82100 6.39000 4.72000 +H 15.51100 7.00000 5.45000 +H 16.32100 6.91000 4.03000 +O 3.54000 15.10100 13.29100 +H 3.33000 14.66100 12.42100 +H 4.51000 15.36100 13.32100 +O 4.02000 7.51000 15.98100 +H 4.70000 8.06000 15.51100 +H 4.42000 6.63000 16.25100 +O 15.87100 7.79000 16.70100 +H 14.95100 8.17000 16.65100 +H 16.47100 8.26000 16.05100 +O 10.13100 1.05000 17.70100 +H 10.19100 1.90000 17.18100 +H 10.45100 0.29000 17.13100 +O 5.04000 0.50000 17.40100 +H 4.62000 -0.07000 16.70100 +H 4.38000 1.19000 17.72100 +O 5.73000 8.70000 10.29100 +H 6.17000 9.59000 10.20100 +H 5.10000 8.70000 11.06100 +O 13.60100 8.62000 10.45100 +H 12.85100 8.62000 9.79100 +H 13.97100 7.70000 10.54100 +O 12.09100 5.25000 2.75000 +H 12.22100 4.41000 3.29000 +H 11.80100 5.99000 3.35000 +O 3.07000 2.13000 12.31100 +H 2.84000 2.50000 13.21100 +H 2.77000 1.18000 12.25100 +O 0.37000 13.10100 12.82100 +H 0.90000 12.61100 13.50100 +H 0.59000 14.08100 12.87100 +O 7.32000 6.34000 10.64100 +H 7.91000 6.08000 9.88100 +H 7.04000 7.30000 10.53100 +O 17.28100 9.35100 18.54100 +H 16.82100 9.28100 17.65100 +H 16.66100 9.79100 19.20100 +O 16.22100 3.67000 3.74000 +H 16.24100 2.91000 4.38000 +H 15.74100 4.44000 4.14000 +O 10.23100 7.66000 9.66100 +H 10.38100 7.87000 10.62100 +H 9.93100 6.71000 9.57100 +O 9.80100 15.73100 17.00100 +H 9.60100 16.17100 16.12100 +H 10.19100 14.82100 16.84100 +O 18.59100 15.18100 16.05100 +H 18.73100 15.45100 15.10100 +H 19.42100 15.40100 16.58100 +O 3.50000 8.98000 18.04100 +H 4.26000 9.42000 18.52100 +H 3.85000 8.51000 17.22100 +O 15.40100 2.74000 1.25000 +H 14.79100 1.99000 1.48000 +H 15.62100 3.26000 2.08000 +O 13.03100 8.38000 0.42000 +H 13.37100 7.45000 0.57000 +H 13.21100 8.65000 -0.53000 +O 10.68100 13.33100 8.49000 +H 10.75100 12.49100 7.94000 +H 11.30100 14.02100 8.13000 +O 3.19000 8.10000 9.49100 +H 4.12000 8.46000 9.54100 +H 3.13000 7.25000 10.01100 +O 3.39000 5.09000 10.06100 +H 2.87000 4.26000 9.89100 +H 4.16000 5.14000 9.42100 +O 11.38100 3.80000 16.78100 +H 10.93100 4.43000 17.42100 +H 12.31100 4.11000 16.61100 +O 11.60100 2.07000 14.77100 +H 11.60100 2.71000 15.54100 +H 11.88100 2.55000 13.94100 +O 5.04000 15.68100 9.10000 +H 5.70000 16.42100 9.19000 +H 5.48000 14.89100 8.68000 +O 10.02100 7.96000 12.38100 +H 10.43100 7.64000 13.24100 +H 9.06100 7.69000 12.35100 +O 0.40000 5.44000 11.14100 +H 1.25000 5.11000 10.73100 +H 0.53000 5.59000 12.12100 +O 1.89000 5.20000 17.22100 +H 2.48000 4.80000 16.52100 +H 1.31000 5.91000 16.81100 +O 13.69100 9.50100 16.60100 +H 14.08100 10.39100 16.80100 +H 13.79100 9.30100 15.63100 +O 8.15000 5.72000 3.25000 +H 8.22000 4.83000 2.79000 +H 7.21000 6.06000 3.17000 +O 16.57100 6.04000 12.06100 +H 16.19100 5.35000 12.68100 +H 17.39100 5.68000 11.62100 +O 2.52000 15.64100 17.44100 +H 2.22000 16.21100 18.20100 +H 2.45000 14.67100 17.70100 +O 6.71000 4.64000 12.69100 +H 6.37000 3.75000 12.39100 +H 6.97000 5.18000 11.89100 +O 9.30000 16.78100 14.65100 +H 9.06000 16.60100 13.70100 +H 9.60000 17.72100 14.75100 +O 4.73000 5.00000 1.91000 +H 5.34000 5.80000 1.95000 +H 3.78000 5.31000 1.98000 +O 1.59000 11.37100 14.66100 +H 1.81000 10.76100 15.42100 +H 1.69000 10.88100 13.80100 +O 13.47100 10.59100 12.34100 +H 13.71100 9.96100 11.60100 +H 12.57100 10.99100 12.16100 +O 13.02100 8.55000 3.09000 +H 12.16100 8.24000 3.51000 +H 12.98100 8.41000 2.10000 +O 17.59100 17.47100 11.54100 +H 18.20100 17.77100 10.81100 +H 17.21100 16.58100 11.32100 +O 12.52100 17.31100 11.28100 +H 13.36100 17.36100 10.74100 +H 12.29100 16.35100 11.46100 +O 0.83000 12.58100 10.22100 +H 0.78000 12.57100 11.22100 +H 0.00000 12.17100 9.84100 +O 6.88000 16.62100 17.16100 +H 6.32000 17.43100 17.25100 +H 7.40000 16.66100 16.30100 +O 9.03000 0.86000 1.33000 +H 9.54000 0.87000 0.47000 +H 9.59000 0.44000 2.04000 +O 17.26100 1.35000 5.23000 +H 17.99100 1.18000 4.56000 +H 16.95100 0.48000 5.61000 +O 13.88100 15.73100 4.77000 +H 14.55100 15.85100 4.03000 +H 13.48100 16.62100 5.00000 +O 1.30000 17.94100 18.51100 +H 0.89000 17.20100 19.04100 +H 1.94000 18.45100 19.09100 +O 12.80100 9.27000 6.72000 +H 13.40100 8.46000 6.74000 +H 13.20100 9.96000 6.12000 +O 8.30000 12.73100 14.22100 +H 8.25000 13.06100 15.17100 +H 7.44000 12.92100 13.76100 +O 6.72000 16.16100 1.54000 +H 6.81000 16.26100 0.55000 +H 6.32000 15.27100 1.75000 +O 16.50100 17.20100 13.94100 +H 17.03100 17.30100 13.10100 +H 16.23100 18.10100 14.28100 +O 18.41100 1.75000 9.63100 +H 18.80100 0.90000 9.27100 +H 17.43100 1.77000 9.44100 +O 2.63000 3.26000 7.20000 +H 1.84000 3.77000 6.86000 +H 2.54000 3.11000 8.18000 +O 11.94100 16.12100 0.31000 +H 12.00100 15.19100 0.68000 +H 11.35100 16.12100 -0.49000 +O 8.22000 10.02100 13.72100 +H 8.62000 10.01100 12.80100 +H 8.32000 10.94100 14.12100 +O 9.16000 9.10000 2.91000 +H 9.79000 9.48000 2.23000 +H 9.56000 8.27000 3.30000 +O 15.04100 16.07100 0.44000 +H 14.12100 16.44100 0.51000 +H 15.42100 16.27100 -0.46000 +O 3.72000 12.88100 14.90100 +H 3.59000 13.81100 14.56100 +H 2.88000 12.36100 14.77100 +O 10.39100 10.98100 6.96000 +H 9.69100 10.51100 7.50000 +H 10.98100 10.30100 6.53000 +O 10.14100 2.36000 9.71100 +H 10.06100 2.00000 8.78100 +H 10.12100 1.60000 10.36100 +O 5.90000 14.87100 4.91000 +H 6.32000 14.29100 4.21000 +H 5.46000 15.66100 4.47000 +O 17.09100 3.85000 13.81100 +H 17.82100 4.54000 13.85100 +H 17.37100 3.10000 13.22100 +O 2.55000 13.48100 2.90000 +H 1.59000 13.49100 2.63000 +H 2.67000 14.01100 3.74000 +O 6.72000 2.03000 14.89100 +H 7.62000 1.87000 14.49100 +H 6.80000 2.08000 15.88100 +O 14.40100 8.56000 13.98100 +H 13.83100 9.08000 13.35100 +H 15.36100 8.68000 13.74100 +O 0.72000 1.66000 3.18000 +H 0.55000 2.49000 2.64000 +H 1.62000 1.29000 2.96000 +O 11.83100 13.35100 1.19000 +H 10.84100 13.24100 1.21000 +H 12.17100 13.50100 2.12000 +O 6.13000 8.42000 14.31100 +H 6.69000 9.23000 14.14100 +H 6.72000 7.62000 14.34100 +O 14.93100 17.67100 9.59100 +H 15.26100 18.31100 8.90100 +H 15.59100 17.61100 10.34100 +O 7.16000 5.65000 17.08100 +H 7.35000 6.30000 17.82100 +H 7.76000 4.85000 17.17100 +O 14.50100 12.20100 16.33100 +H 14.41100 12.10100 17.32100 +H 15.46100 12.13100 16.07100 +O 3.90000 17.41100 15.60100 +H 2.99000 17.82100 15.58100 +H 3.83000 16.47100 15.92100 +O 16.74100 8.83000 12.54100 +H 16.47100 7.94000 12.17100 +H 16.75100 9.51000 11.81100 +O 12.25100 3.25000 4.49000 +H 12.90100 2.51000 4.38000 +H 12.45100 3.75000 5.33000 +O 5.94000 7.45000 6.52000 +H 6.44000 8.30000 6.33000 +H 5.06000 7.47000 6.04000 +O 17.77100 3.42000 16.42100 +H 17.60100 3.73000 15.48100 +H 16.93100 3.05000 16.80100 +O 17.30100 9.34100 15.17100 +H 17.68100 10.25100 15.32100 +H 17.22100 9.17100 14.18100 +O 8.59000 13.74100 0.16000 +H 8.13000 13.89100 1.04000 +H 9.03000 14.59100 -0.14000 +O 6.61000 17.90100 9.53100 +H 6.15000 18.78100 9.40100 +H 7.60000 18.02100 9.46100 +O 14.08100 18.51100 17.20100 +H 13.12100 18.40100 16.93100 +H 14.64100 17.84100 16.72100 +O 8.59000 9.56100 8.61000 +H 9.13000 8.87100 9.09000 +H 8.27000 10.25100 9.27000 +O 10.83100 9.84100 0.87000 +H 10.60100 10.37100 0.05000 +H 11.64100 9.28100 0.68000 +O 18.61100 15.69100 8.51000 +H 17.90100 15.57100 7.81000 +H 18.62100 14.90100 9.11000 +O 2.21000 13.14100 18.44100 +H 1.56000 12.41100 18.23100 +H 2.25000 13.28100 19.42100 +O 0.79000 12.40100 6.53000 +H 0.78000 11.93100 7.41000 +H 1.61000 12.12100 6.02000 +O 6.72000 13.91100 16.24100 +H 5.94000 13.41100 16.62100 +H 6.69000 14.86100 16.55100 +O 18.24100 1.92000 12.27100 +H 18.20100 1.02000 12.71100 +H 18.27100 1.81000 11.28100 +O 4.28000 4.24000 5.20000 +H 4.58000 3.52000 4.58000 +H 3.89000 3.84000 6.03000 +O 17.05100 14.87100 11.04100 +H 16.12100 14.62100 10.77100 +H 17.31100 14.37100 11.86100 +O 3.17000 5.47000 12.80100 +H 3.55000 4.88000 13.52100 +H 3.57000 5.21000 11.92100 +O 8.12000 15.86100 6.87000 +H 8.44000 15.96100 5.93000 +H 7.33000 15.24100 6.89000 +O 14.24100 2.14000 11.12100 +H 14.76100 1.49000 11.67100 +H 13.75100 2.77000 11.73100 +O 10.01100 0.34000 11.54100 +H 9.38100 -0.38000 11.23100 +H 10.94100 -0.02000 11.54100 +O 7.70000 13.30100 3.01000 +H 7.24000 12.43100 3.18000 +H 8.61000 13.27100 3.42000 +O 6.18000 15.67100 12.84100 +H 6.13000 16.49100 13.41100 +H 7.07000 15.64100 12.39100 +O 13.52100 0.52000 1.68000 +H 13.87100 0.11000 0.84000 +H 12.62100 0.14000 1.88000 +O 13.00100 4.53000 6.91000 +H 12.41100 5.33000 6.95000 +H 13.15100 4.18000 7.84000 +O 15.93100 2.21000 8.82000 +H 15.09100 2.20000 9.36000 +H 15.95100 3.04000 8.26000 +O 0.39000 10.77100 3.00000 +H 1.38000 10.66100 2.91000 +H -0.01000 9.91100 3.32000 +O 8.75000 16.46100 3.37000 +H 7.98000 16.11100 2.83000 +H 8.43000 17.17100 3.99000 +O 2.30000 6.28000 19.75100 +H 1.37000 6.26000 20.12100 +H 2.31000 5.89000 18.83100 +O 15.55100 15.11100 25.65100 +H 14.98100 14.95100 26.46100 +H 14.96100 15.21100 24.85100 +O 17.43100 6.18000 27.18100 +H 17.76100 7.12000 27.18100 +H 17.94100 5.64000 27.84100 +O 11.35100 7.03000 25.79100 +H 11.92100 7.81000 25.54100 +H 10.75100 7.29000 26.55100 +O 17.55100 6.07000 20.93100 +H 17.43100 5.94000 19.94100 +H 17.25100 5.26000 21.42100 +O 7.68000 11.44100 1.02100 +H 6.90000 11.61100 1.62100 +H 8.02000 12.31100 0.66100 +O 6.85000 10.12100 25.27100 +H 7.54000 9.96100 25.97100 +H 6.12000 10.69100 25.65100 +O 2.31000 17.13100 23.45100 +H 2.65000 17.90100 23.99100 +H 2.75000 17.13100 22.55100 +O 2.40000 10.91100 27.48100 +H 2.54000 10.07100 28.00100 +H 1.85000 11.55100 28.03100 +O 6.06000 9.64100 19.85100 +H 6.13000 10.48100 19.31100 +H 6.52000 9.77100 20.73100 +O 18.42100 17.67100 22.21100 +H 18.96100 17.38100 23.01100 +H 18.72100 18.57100 21.92100 +O 16.89100 9.22000 24.74100 +H 17.84100 8.93000 24.82100 +H 16.81100 9.87000 23.99100 +O 16.41100 11.08100 22.94100 +H 17.27100 11.10100 22.42100 +H 16.55100 11.55100 23.82100 +O 6.13000 13.65100 25.88100 +H 5.64000 12.78100 25.97100 +H 5.90000 14.08100 25.01100 +O 8.09000 0.04000 23.64100 +H 8.49000 0.95000 23.55100 +H 7.09000 0.12000 23.70100 +O 15.25100 9.99100 20.52100 +H 14.62100 9.23100 20.65100 +H 15.73100 10.17100 21.38100 +O 16.67100 17.14100 24.34100 +H 16.26100 16.91100 23.46100 +H 16.49100 16.40100 24.99100 +O 5.98000 7.29000 21.32100 +H 6.22000 7.98000 20.64100 +H 5.20000 7.62000 21.86100 +O 12.81100 3.45000 0.23100 +H 11.95100 2.95000 0.10100 +H 13.43100 2.91000 0.79100 +O 15.76100 16.62100 21.69100 +H 16.65100 17.08100 21.72100 +H 15.55100 16.38100 20.74100 +O 10.41100 7.01000 22.91100 +H 10.67100 6.97000 23.87100 +H 9.56100 6.50000 22.77100 +O 8.65000 3.48000 20.57100 +H 9.24000 4.11000 20.08100 +H 8.84000 2.54000 20.28100 +O 13.62100 11.44100 24.07100 +H 14.45100 11.15100 23.59100 +H 13.13100 12.11100 23.51100 +O 10.08100 14.56100 23.39100 +H 9.62100 15.28100 22.87100 +H 10.04100 14.76100 24.37100 +O 3.51000 18.01100 27.15100 +H 4.01000 17.15100 27.21100 +H 4.16000 18.78100 27.12100 +O 17.95100 10.66100 27.35100 +H 17.33100 10.51100 26.59100 +H 17.43100 10.77100 28.20100 +O 3.21000 9.43100 21.04100 +H 4.03000 9.82100 20.62100 +H 2.94000 8.61100 20.55100 +O 14.58100 7.35000 25.90100 +H 14.53100 6.70000 26.65100 +H 15.38100 7.94000 26.03100 +O 11.11100 17.76100 20.99100 +H 10.51100 17.14100 21.49100 +H 11.42100 17.32100 20.14100 +O 16.32100 13.77100 19.43100 +H 16.00100 14.71100 19.33100 +H 15.56100 13.14100 19.31100 +O 4.64000 17.43100 21.85100 +H 4.97000 17.82100 22.71100 +H 5.40000 17.36100 21.20100 +O 14.00100 1.07000 22.88100 +H 13.76100 0.70000 21.98100 +H 14.99100 1.23000 22.92100 +O 3.82000 7.00000 23.42100 +H 4.27000 6.10000 23.39100 +H 2.88000 6.89000 23.75100 +O 6.14000 1.22000 19.79100 +H 7.12000 1.00000 19.86100 +H 5.83000 1.05000 18.86100 +O 5.39000 0.64000 23.74100 +H 4.58000 0.65000 24.32100 +H 5.42000 1.47000 23.19100 +O 14.34100 11.88100 19.03100 +H 14.66100 11.12100 19.60100 +H 13.42100 12.15100 19.33100 +O 9.35100 2.36000 23.42100 +H 8.87100 2.77000 22.64100 +H 10.34100 2.34000 23.23100 +O 4.59000 11.52100 26.03100 +H 3.88000 11.25100 26.68100 +H 4.33000 11.24100 25.10100 +O 12.71100 17.97100 24.53100 +H 13.15100 18.61100 23.89100 +H 12.21100 18.49100 25.23100 +O 10.32100 5.49000 18.78100 +H 9.91100 6.31000 18.39100 +H 10.96100 5.75000 19.51100 +O 8.66000 4.54000 25.04100 +H 8.34000 5.26000 24.42100 +H 8.90000 3.73000 24.51100 +O 10.17100 0.39000 26.15100 +H 9.45100 0.44000 25.46100 +H 9.93100 -0.30000 26.84100 +O 18.57100 8.33000 22.39100 +H 18.99100 7.69000 23.03100 +H 18.19100 7.82000 21.61100 +O 4.88000 13.85100 20.36100 +H 4.01000 13.70100 20.83100 +H 4.71000 14.11100 19.41100 +O 16.64100 12.80100 25.19100 +H 17.63100 12.88100 25.33100 +H 16.19100 13.64100 25.50100 +O 13.90100 5.75000 19.40100 +H 13.36100 5.54000 20.21100 +H 14.81100 5.34000 19.49100 +O 5.27000 2.56000 21.90100 +H 5.54000 1.97000 21.15100 +H 5.27000 3.51000 21.59100 +O 7.05000 10.50100 22.30100 +H 6.91000 10.57100 23.29100 +H 7.89000 9.99100 22.12100 +O 3.67000 11.00100 23.63100 +H 3.60000 11.83100 23.07100 +H 3.71000 10.20100 23.03100 +O 5.66000 5.37000 27.27100 +H 5.78000 6.03000 26.53100 +H 6.12000 5.71000 28.10100 +O 12.52100 13.48100 22.50100 +H 13.02100 14.25100 22.90100 +H 11.57100 13.50100 22.82100 +O 15.82100 6.39000 23.34100 +H 15.51100 7.00000 24.07100 +H 16.32100 6.91000 22.65100 +O 12.09100 5.25000 21.37100 +H 12.22100 4.41000 21.91100 +H 11.80100 5.99000 21.97100 +O 16.22100 3.67000 22.36100 +H 16.24100 2.91000 23.00100 +H 15.74100 4.44000 22.76100 +O 15.40100 2.74000 19.87100 +H 14.79100 1.99000 20.10100 +H 15.62100 3.26000 20.70100 +O 13.03100 8.38000 19.04100 +H 13.37100 7.45000 19.19100 +H 13.21100 8.65000 18.09100 +O 10.68100 13.33100 27.11100 +H 10.75100 12.49100 26.56100 +H 11.30100 14.02100 26.75100 +O 8.15000 5.72000 21.87100 +H 8.22000 4.83000 21.41100 +H 7.21000 6.06000 21.79100 +O 4.73000 5.00000 20.53100 +H 5.34000 5.80000 20.57100 +H 3.78000 5.31000 20.60100 +O 13.02100 8.55000 21.71100 +H 12.16100 8.24000 22.13100 +H 12.98100 8.41000 20.72100 +O 17.26100 1.35000 23.85100 +H 17.99100 1.18000 23.18100 +H 16.95100 0.48000 24.23100 +O 13.88100 15.73100 23.39100 +H 14.55100 15.85100 22.65100 +H 13.48100 16.62100 23.62100 +O 12.80100 9.27000 25.34100 +H 13.40100 8.46000 25.36100 +H 13.20100 9.96000 24.74100 +O 6.72000 16.16100 20.16100 +H 6.81000 16.26100 19.17100 +H 6.32000 15.27100 20.37100 +O 18.41100 1.75000 0.42100 +H 18.80100 0.90000 0.06100 +H 17.43100 1.77000 0.23100 +O 2.63000 3.26000 25.82100 +H 1.84000 3.77000 25.48100 +H 2.54000 3.11000 26.80100 +O 11.94100 16.12100 18.93100 +H 12.00100 15.19100 19.30100 +H 11.35100 16.12100 18.13100 +O 9.16000 9.10000 21.53100 +H 9.79000 9.48000 20.85100 +H 9.56000 8.27000 21.92100 +O 15.04100 16.07100 19.06100 +H 14.12100 16.44100 19.13100 +H 15.42100 16.27100 18.16100 +O 10.39100 10.98100 25.58100 +H 9.69100 10.51100 26.12100 +H 10.98100 10.30100 25.15100 +O 5.90000 14.87100 23.53100 +H 6.32000 14.29100 22.83100 +H 5.46000 15.66100 23.09100 +O 2.55000 13.48100 21.52100 +H 1.59000 13.49100 21.25100 +H 2.67000 14.01100 22.36100 +O 0.75000 3.45000 18.95100 +H -0.17000 3.17000 18.66100 +H 1.06000 4.22000 18.39100 +O 0.72000 1.66000 21.80100 +H 0.55000 2.49000 21.26100 +H 1.62000 1.29000 21.58100 +O 11.83100 13.35100 19.81100 +H 10.84100 13.24100 19.83100 +H 12.17100 13.50100 20.74100 +O 12.25100 3.25000 23.11100 +H 12.90100 2.51000 23.00100 +H 12.45100 3.75000 23.95100 +O 5.94000 7.45000 25.14100 +H 6.44000 8.30000 24.95100 +H 5.06000 7.47000 24.66100 +O 8.59000 13.74100 18.78100 +H 8.13000 13.89100 19.66100 +H 9.03000 14.59100 18.48100 +O 6.61000 17.90100 0.32100 +H 6.15000 18.78100 0.19100 +H 7.60000 18.02100 0.25100 +O 8.59000 9.56100 27.23100 +H 9.13000 8.87100 27.71100 +H 8.27000 10.25100 27.89100 +O 10.83100 9.84100 19.49100 +H 10.60100 10.37100 18.67100 +H 11.64100 9.28100 19.30100 +O 18.61100 15.69100 27.13100 +H 17.90100 15.57100 26.43100 +H 18.62100 14.90100 27.73100 +O 0.79000 12.40100 25.15100 +H 0.78000 11.93100 26.03100 +H 1.61000 12.12100 24.64100 +O 4.28000 4.24000 23.82100 +H 4.58000 3.52000 23.20100 +H 3.89000 3.84000 24.65100 +O 8.12000 15.86100 25.49100 +H 8.44000 15.96100 24.55100 +H 7.33000 15.24100 25.51100 +O 7.70000 13.30100 21.63100 +H 7.24000 12.43100 21.80100 +H 8.61000 13.27100 22.04100 +O 13.52100 0.52000 20.30100 +H 13.87100 0.11000 19.46100 +H 12.62100 0.14000 20.50100 +O 13.00100 4.53000 25.53100 +H 12.41100 5.33000 25.57100 +H 13.15100 4.18000 26.46100 +O 0.39000 10.77100 21.62100 +H 1.38000 10.66100 21.53100 +H -0.01000 9.91100 21.94100 +O 8.75000 16.46100 21.99100 +H 7.98000 16.11100 21.45100 +H 8.43000 17.17100 22.61100 +O 2.30000 24.90100 1.13000 +H 1.37000 24.88100 1.50000 +H 2.31000 24.51100 0.21000 +O 17.43100 24.80100 8.56000 +H 17.76100 25.74100 8.56000 +H 17.94100 24.26100 9.22000 +O 11.35100 25.65100 7.17000 +H 11.92100 26.43100 6.92000 +H 10.75100 25.91100 7.93000 +O 17.55100 24.69100 2.31000 +H 17.43100 24.56100 1.32000 +H 17.25100 23.88100 2.80000 +O 8.50000 26.60100 18.23100 +H 8.46000 27.36100 18.88100 +H 8.72000 26.96100 17.32100 +O 6.85000 0.91100 6.65000 +H 7.54000 0.75100 7.35000 +H 6.12000 1.48100 7.03000 +O 4.38000 22.54100 14.99100 +H 5.20000 21.98100 15.08100 +H 3.57000 21.96100 15.03100 +O 15.94100 19.76100 14.80100 +H 15.76100 20.43100 14.08100 +H 15.91100 20.22100 15.69100 +O 1.13000 25.99100 15.97100 +H 2.01000 25.86100 16.42100 +H 1.00000 26.96100 15.75100 +O 8.09000 18.66100 5.02000 +H 8.49000 19.57100 4.93000 +H 7.09000 18.74100 5.08000 +O 3.17000 21.13100 18.01100 +H 3.88000 21.84100 18.07100 +H 2.29000 21.52100 18.29100 +O 5.98000 25.91100 2.70000 +H 6.22000 26.60100 2.02000 +H 5.20000 26.24100 3.24000 +O 12.81100 22.07100 9.44100 +H 11.95100 21.57100 9.31100 +H 13.43100 21.53100 10.00100 +O 8.07000 24.67100 14.65100 +H 7.60000 24.64100 15.54100 +H 7.56000 24.12100 13.99100 +O 13.94100 23.31100 16.74100 +H 13.74100 23.74100 17.62100 +H 14.72100 22.69100 16.83100 +O 9.91100 22.72100 12.42100 +H 9.14100 23.06100 12.96100 +H 9.57100 22.21100 11.63100 +O 10.41100 25.63100 4.29000 +H 10.67100 25.59100 5.25000 +H 9.56100 25.12100 4.15000 +O 8.65000 22.10100 1.95000 +H 9.24000 22.73100 1.46000 +H 8.84000 21.16100 1.66000 +O 17.19100 24.47100 18.31100 +H 16.93100 25.36100 17.95100 +H 17.17100 23.79100 17.58100 +O 5.50000 20.58100 8.85000 +H 5.45000 20.53100 9.85000 +H 5.52000 21.54100 8.56000 +O 14.58100 25.97100 7.28000 +H 14.53100 25.32100 8.03000 +H 15.38100 26.56100 7.41000 +O 2.02000 21.47100 14.98100 +H 1.22000 22.07100 14.85100 +H 1.92000 20.98100 15.84100 +O 14.00100 19.69100 4.26000 +H 13.76100 19.32100 3.36000 +H 14.99100 19.85100 4.30000 +O 3.82000 25.62100 4.80000 +H 4.27000 24.72100 4.77000 +H 2.88000 25.51100 5.13000 +O 15.47100 20.84100 17.29100 +H 15.42100 21.21100 18.21100 +H 14.75100 20.15100 17.17100 +O 7.81000 21.26100 17.49100 +H 8.48000 20.65100 17.92100 +H 7.08000 21.45100 18.14100 +O 13.51100 24.52100 14.33100 +H 13.79100 24.09100 15.18100 +H 13.76100 25.48100 14.34100 +O 9.22000 23.65100 8.99000 +H 8.97000 23.56100 8.03000 +H 9.70000 22.83100 9.30000 +O 5.39000 19.26100 5.12000 +H 4.58000 19.27100 5.70000 +H 5.42000 20.09100 4.57000 +O 2.97000 18.97100 1.71000 +H 3.46000 19.81100 1.50000 +H 3.59000 18.32100 2.16000 +O 9.35100 20.98100 4.80000 +H 8.87100 21.39100 4.02000 +H 10.34100 20.96100 4.61000 +O 10.76100 25.45100 14.64100 +H 9.96100 24.84100 14.67100 +H 11.57100 24.92100 14.40100 +O 4.59000 2.31100 7.41000 +H 3.88000 2.04100 8.06000 +H 4.33000 2.03100 6.48000 +O 10.32100 24.11100 0.16000 +H 9.91100 24.93100 -0.23000 +H 10.96100 24.37100 0.89000 +O 0.78000 24.18100 13.86100 +H 1.70000 24.17100 13.45100 +H 0.72000 24.92100 14.53100 +O 5.61000 20.84100 11.47100 +H 5.99000 20.00100 11.84100 +H 4.73000 21.03100 11.91100 +O 8.66000 23.16100 6.42000 +H 8.34000 23.88100 5.80000 +H 8.90000 22.35100 5.89000 +O 10.17100 19.01100 7.53000 +H 9.45100 19.06100 6.84000 +H 9.93100 18.32100 8.22000 +O 14.66100 24.52100 9.92100 +H 14.36100 23.57100 9.99100 +H 15.39100 24.68100 10.58100 +O 18.57100 26.95100 3.77000 +H 18.99100 26.31100 4.41000 +H 18.19100 26.44100 2.99000 +O 13.90100 24.37100 0.78000 +H 13.36100 24.16100 1.59000 +H 14.81100 23.96100 0.87000 +O 5.27000 21.18100 3.28000 +H 5.54000 20.59100 2.53000 +H 5.27000 22.13100 2.97000 +O 9.15000 19.51100 14.02100 +H 9.40000 19.31100 13.07100 +H 9.87000 20.07100 14.44100 +O 4.10000 26.75100 12.51100 +H 4.96000 26.87100 13.01100 +H 3.68000 25.88100 12.78100 +O 12.74100 22.48100 12.62100 +H 12.95100 23.22100 13.26100 +H 11.85100 22.65100 12.19100 +O 5.66000 23.99100 8.65000 +H 5.78000 24.65100 7.91000 +H 6.12000 24.33100 9.48000 +O 15.82100 25.01100 4.72000 +H 15.51100 25.62100 5.45000 +H 16.32100 25.53100 4.03000 +O 4.02000 26.13100 15.98100 +H 4.70000 26.68100 15.51100 +H 4.42000 25.25100 16.25100 +O 15.87100 26.41100 16.70100 +H 14.95100 26.79100 16.65100 +H 16.47100 26.88100 16.05100 +O 10.13100 19.67100 17.70100 +H 10.19100 20.52100 17.18100 +H 10.45100 18.91100 17.13100 +O 5.04000 19.12100 17.40100 +H 4.62000 18.55100 16.70100 +H 4.38000 19.81100 17.72100 +O 5.73000 27.32100 10.29100 +H 6.17000 28.21100 10.20100 +H 5.10000 27.32100 11.06100 +O 13.60100 27.24100 10.45100 +H 12.85100 27.24100 9.79100 +H 13.97100 26.32100 10.54100 +O 12.09100 23.87100 2.75000 +H 12.22100 23.03100 3.29000 +H 11.80100 24.61100 3.35000 +O 3.07000 20.75100 12.31100 +H 2.84000 21.12100 13.21100 +H 2.77000 19.80100 12.25100 +O 7.32000 24.96100 10.64100 +H 7.91000 24.70100 9.88100 +H 7.04000 25.92100 10.53100 +O 17.28100 0.14100 18.54100 +H 16.82100 0.07100 17.65100 +H 16.66100 0.58100 19.20100 +O 3.07000 19.25100 6.18000 +H 2.96000 20.19100 6.51000 +H 3.02000 18.62100 6.95000 +O 16.22100 22.29100 3.74000 +H 16.24100 21.53100 4.38000 +H 15.74100 23.06100 4.14000 +O 10.23100 26.28100 9.66100 +H 10.38100 26.49100 10.62100 +H 9.93100 25.33100 9.57100 +O 15.40100 21.36100 1.25000 +H 14.79100 20.61100 1.48000 +H 15.62100 21.88100 2.08000 +O 3.19000 26.72100 9.49100 +H 4.12000 27.08100 9.54100 +H 3.13000 25.87100 10.01100 +O 3.39000 23.71100 10.06100 +H 2.87000 22.88100 9.89100 +H 4.16000 23.76100 9.42100 +O 11.38100 22.42100 16.78100 +H 10.93100 23.05100 17.42100 +H 12.31100 22.73100 16.61100 +O 11.60100 20.69100 14.77100 +H 11.60100 21.33100 15.54100 +H 11.88100 21.17100 13.94100 +O 10.02100 26.58100 12.38100 +H 10.43100 26.26100 13.24100 +H 9.06100 26.31100 12.35100 +O 0.40000 24.06100 11.14100 +H 1.25000 23.73100 10.73100 +H 0.53000 24.21100 12.12100 +O 1.89000 23.82100 17.22100 +H 2.48000 23.42100 16.52100 +H 1.31000 24.53100 16.81100 +O 8.15000 24.34100 3.25000 +H 8.22000 23.45100 2.79000 +H 7.21000 24.68100 3.17000 +O 16.57100 24.66100 12.06100 +H 16.19100 23.97100 12.68100 +H 17.39100 24.30100 11.62100 +O 6.71000 23.26100 12.69100 +H 6.37000 22.37100 12.39100 +H 6.97000 23.80100 11.89100 +O 4.73000 23.62100 1.91000 +H 5.34000 24.42100 1.95000 +H 3.78000 23.93100 1.98000 +O 9.03000 19.48100 1.33000 +H 9.54000 19.49100 0.47000 +H 9.59000 19.06100 2.04000 +O 17.26100 19.97100 5.23000 +H 17.99100 19.80100 4.56000 +H 16.95100 19.10100 5.61000 +O 12.80100 0.06100 6.72000 +H 13.40100 -0.74900 6.74000 +H 13.20100 0.75100 6.12000 +O 18.41100 20.37100 9.63100 +H 18.80100 19.52100 9.27100 +H 17.43100 20.39100 9.44100 +O 2.63000 21.88100 7.20000 +H 1.84000 22.39100 6.86000 +H 2.54000 21.73100 8.18000 +O 10.14100 20.98100 9.71100 +H 10.06100 20.62100 8.78100 +H 10.12100 20.22100 10.36100 +O 17.09100 22.47100 13.81100 +H 17.82100 23.16100 13.85100 +H 17.37100 21.72100 13.22100 +O 6.72000 20.65100 14.89100 +H 7.62000 20.49100 14.49100 +H 6.80000 20.70100 15.88100 +O 14.40100 27.18100 13.98100 +H 13.83100 27.70100 13.35100 +H 15.36100 27.30100 13.74100 +O 0.72000 20.28100 3.18000 +H 0.55000 21.11100 2.64000 +H 1.62000 19.91100 2.96000 +O 6.13000 27.04100 14.31100 +H 6.69000 27.85100 14.14100 +H 6.72000 26.24100 14.34100 +O 7.16000 24.27100 17.08100 +H 7.35000 24.92100 17.82100 +H 7.76000 23.47100 17.17100 +O 16.74100 27.45100 12.54100 +H 16.47100 26.56100 12.17100 +H 16.75100 28.13100 11.81100 +O 12.25100 21.87100 4.49000 +H 12.90100 21.13100 4.38000 +H 12.45100 22.37100 5.33000 +O 5.94000 26.07100 6.52000 +H 6.44000 26.92100 6.33000 +H 5.06000 26.09100 6.04000 +O 17.77100 22.04100 16.42100 +H 17.60100 22.35100 15.48100 +H 16.93100 21.67100 16.80100 +O 17.30100 0.13100 15.17100 +H 17.68100 1.04100 15.32100 +H 17.22100 -0.03900 14.18100 +O 18.24100 20.54100 12.27100 +H 18.20100 19.64100 12.71100 +H 18.27100 20.43100 11.28100 +O 4.28000 22.86100 5.20000 +H 4.58000 22.14100 4.58000 +H 3.89000 22.46100 6.03000 +O 3.17000 24.09100 12.80100 +H 3.55000 23.50100 13.52100 +H 3.57000 23.83100 11.92100 +O 14.24100 20.76100 11.12100 +H 14.76100 20.11100 11.67100 +H 13.75100 21.39100 11.73100 +O 10.01100 18.96100 11.54100 +H 9.38100 18.24100 11.23100 +H 10.94100 18.60100 11.54100 +O 13.52100 19.14100 1.68000 +H 13.87100 18.73100 0.84000 +H 12.62100 18.76100 1.88000 +O 13.00100 23.15100 6.91000 +H 12.41100 23.95100 6.95000 +H 13.15100 22.80100 7.84000 +O 15.93100 20.83100 8.82000 +H 15.09100 20.82100 9.36000 +H 15.95100 21.66100 8.26000 +O 2.30000 24.90100 19.75100 +H 1.37000 24.88100 20.12100 +H 2.31000 24.51100 18.83100 +O 17.43100 24.80100 27.18100 +H 17.76100 25.74100 27.18100 +H 17.94100 24.26100 27.84100 +O 11.35100 25.65100 25.79100 +H 11.92100 26.43100 25.54100 +H 10.75100 25.91100 26.55100 +O 17.55100 24.69100 20.93100 +H 17.43100 24.56100 19.94100 +H 17.25100 23.88100 21.42100 +O 6.85000 0.91100 25.27100 +H 7.54000 0.75100 25.97100 +H 6.12000 1.48100 25.65100 +O 2.40000 1.70100 27.48100 +H 2.54000 0.86100 28.00100 +H 1.85000 2.34100 28.03100 +O 8.09000 18.66100 23.64100 +H 8.49000 19.57100 23.55100 +H 7.09000 18.74100 23.70100 +O 5.98000 25.91100 21.32100 +H 6.22000 26.60100 20.64100 +H 5.20000 26.24100 21.86100 +O 12.81100 22.07100 0.23100 +H 11.95100 21.57100 0.10100 +H 13.43100 21.53100 0.79100 +O 10.41100 25.63100 22.91100 +H 10.67100 25.59100 23.87100 +H 9.56100 25.12100 22.77100 +O 0.76000 26.73100 26.51100 +H 1.75000 26.61100 26.60100 +H 0.52000 27.68100 26.72100 +O 8.65000 22.10100 20.57100 +H 9.24000 22.73100 20.08100 +H 8.84000 21.16100 20.28100 +O 5.50000 20.58100 27.47100 +H 5.45000 20.53100 28.47100 +H 5.52000 21.54100 27.18100 +O 14.58100 25.97100 25.90100 +H 14.53100 25.32100 26.65100 +H 15.38100 26.56100 26.03100 +O 14.00100 19.69100 22.88100 +H 13.76100 19.32100 21.98100 +H 14.99100 19.85100 22.92100 +O 3.82000 25.62100 23.42100 +H 4.27000 24.72100 23.39100 +H 2.88000 25.51100 23.75100 +O 6.14000 19.84100 19.79100 +H 7.12000 19.62100 19.86100 +H 5.83000 19.67100 18.86100 +O 5.39000 19.26100 23.74100 +H 4.58000 19.27100 24.32100 +H 5.42000 20.09100 23.19100 +O 2.97000 18.97100 20.33100 +H 3.46000 19.81100 20.12100 +H 3.59000 18.32100 20.78100 +O 9.35100 20.98100 23.42100 +H 8.87100 21.39100 22.64100 +H 10.34100 20.96100 23.23100 +O 4.59000 2.31100 26.03100 +H 3.88000 2.04100 26.68100 +H 4.33000 2.03100 25.10100 +O 10.32100 24.11100 18.78100 +H 9.91100 24.93100 18.39100 +H 10.96100 24.37100 19.51100 +O 8.66000 23.16100 25.04100 +H 8.34000 23.88100 24.42100 +H 8.90000 22.35100 24.51100 +O 10.17100 19.01100 26.15100 +H 9.45100 19.06100 25.46100 +H 9.93100 18.32100 26.84100 +O 18.57100 26.95100 22.39100 +H 18.99100 26.31100 23.03100 +H 18.19100 26.44100 21.61100 +O 13.90100 24.37100 19.40100 +H 13.36100 24.16100 20.21100 +H 14.81100 23.96100 19.49100 +O 5.27000 21.18100 21.90100 +H 5.54000 20.59100 21.15100 +H 5.27000 22.13100 21.59100 +O 5.66000 23.99100 27.27100 +H 5.78000 24.65100 26.53100 +H 6.12000 24.33100 28.10100 +O 15.82100 25.01100 23.34100 +H 15.51100 25.62100 24.07100 +H 16.32100 25.53100 22.65100 +O 12.09100 23.87100 21.37100 +H 12.22100 23.03100 21.91100 +H 11.80100 24.61100 21.97100 +O 3.07000 19.25100 24.80100 +H 2.96000 20.19100 25.13100 +H 3.02000 18.62100 25.57100 +O 16.22100 22.29100 22.36100 +H 16.24100 21.53100 23.00100 +H 15.74100 23.06100 22.76100 +O 15.40100 21.36100 19.87100 +H 14.79100 20.61100 20.10100 +H 15.62100 21.88100 20.70100 +O 8.15000 24.34100 21.87100 +H 8.22000 23.45100 21.41100 +H 7.21000 24.68100 21.79100 +O 4.73000 23.62100 20.53100 +H 5.34000 24.42100 20.57100 +H 3.78000 23.93100 20.60100 +O 9.03000 19.48100 19.95100 +H 9.54000 19.49100 19.09100 +H 9.59000 19.06100 20.66100 +O 17.26100 19.97100 23.85100 +H 17.99100 19.80100 23.18100 +H 16.95100 19.10100 24.23100 +O 12.80100 0.06100 25.34100 +H 13.40100 -0.74900 25.36100 +H 13.20100 0.75100 24.74100 +O 18.41100 20.37100 0.42100 +H 18.80100 19.52100 0.06100 +H 17.43100 20.39100 0.23100 +O 2.63000 21.88100 25.82100 +H 1.84000 22.39100 25.48100 +H 2.54000 21.73100 26.80100 +O 9.16000 27.72100 21.53100 +H 9.79000 28.10100 20.85100 +H 9.56000 26.89100 21.92100 +O 0.75000 22.07100 18.95100 +H -0.17000 21.79100 18.66100 +H 1.06000 22.84100 18.39100 +O 0.72000 20.28100 21.80100 +H 0.55000 21.11100 21.26100 +H 1.62000 19.91100 21.58100 +O 12.25100 21.87100 23.11100 +H 12.90100 21.13100 23.00100 +H 12.45100 22.37100 23.95100 +O 5.94000 26.07100 25.14100 +H 6.44000 26.92100 24.95100 +H 5.06000 26.09100 24.66100 +O 4.28000 22.86100 23.82100 +H 4.58000 22.14100 23.20100 +H 3.89000 22.46100 24.65100 +O 13.52100 19.14100 20.30100 +H 13.87100 18.73100 19.46100 +H 12.62100 18.76100 20.50100 +O 13.00100 23.15100 25.53100 +H 12.41100 23.95100 25.57100 +H 13.15100 22.80100 26.46100 +O 20.92100 6.28000 1.13000 +H 19.99100 6.26000 1.50000 +H 20.93100 5.89000 0.21000 +O 20.87100 2.75000 9.96100 +H 21.22100 2.58000 10.88100 +H 19.99100 2.30000 9.84100 +O 18.81100 3.68000 6.47000 +H 17.99100 4.11000 6.86000 +H 18.53100 2.95000 5.84000 +O 24.31100 12.75100 11.65100 +H 23.38100 12.68100 11.28100 +H 24.42100 13.64100 12.09100 +O 27.12100 7.98000 18.23100 +H 27.08100 8.74000 18.88100 +H 27.34100 8.34000 17.32100 +O 25.47100 10.12100 6.65000 +H 26.16100 9.96100 7.35000 +H 24.74100 10.69100 7.03000 +O 25.48100 11.61100 18.03100 +H 26.08100 12.40100 18.17100 +H 24.62100 11.92100 17.62100 +O 21.97100 14.35100 10.61100 +H 21.19100 14.04100 10.08100 +H 22.55100 14.93100 10.04100 +O 23.00100 3.92000 14.99100 +H 23.82100 3.36000 15.08100 +H 22.19100 3.34000 15.03100 +O 20.93100 17.13100 4.83000 +H 21.27100 17.90100 5.37000 +H 21.37100 17.13100 3.93000 +O 20.92100 14.34100 5.38000 +H 20.66100 15.30100 5.38000 +H 20.21100 13.80100 5.83000 +O 21.02100 10.91100 8.86000 +H 21.16100 10.07100 9.38000 +H 20.47100 11.55100 9.41000 +O 24.82100 17.86100 14.39100 +H 23.90100 17.69100 14.74100 +H 25.10100 18.78100 14.65100 +O 24.68100 9.64100 1.23000 +H 24.75100 10.48100 0.69000 +H 25.14100 9.77100 2.11000 +O 19.84100 6.43000 5.63000 +H 19.39100 5.55000 5.80000 +H 19.83100 6.97000 6.47000 +O 18.89100 15.96100 1.17000 +H 18.70100 15.00100 1.38000 +H 18.56100 16.54100 1.92000 +O 19.75100 7.37000 15.97100 +H 20.63100 7.24000 16.42100 +H 19.62100 8.34000 15.75100 +O 24.75100 13.65100 7.26000 +H 24.26100 12.78100 7.35000 +H 24.52100 14.08100 6.39000 +O 26.71100 0.04000 5.02000 +H 27.11100 0.95000 4.93000 +H 25.71100 0.12000 5.08000 +O 20.59100 9.76100 12.64100 +H 21.48100 9.31100 12.50100 +H 19.86100 9.11100 12.45100 +O 21.79100 2.51000 18.01100 +H 22.50100 3.22000 18.07100 +H 20.91100 2.90000 18.29100 +O 24.60100 7.29000 2.70000 +H 24.84100 7.98000 2.02000 +H 23.82100 7.62000 3.24000 +O 26.69100 6.05000 14.65100 +H 26.22100 6.02000 15.54100 +H 26.18100 5.50000 13.99100 +O 0.52100 8.90000 15.72100 +H 0.98100 8.06000 15.43100 +H -0.03900 9.24000 14.97100 +O 1.20100 7.01000 4.29000 +H 1.46100 6.97000 5.25000 +H 0.35100 6.50000 4.15000 +O 19.38100 8.11000 7.89000 +H 20.37100 7.99000 7.98000 +H 19.14100 9.06000 8.10000 +O 19.92100 18.21100 15.71100 +H 19.82100 18.06100 16.70100 +H 19.06100 18.57100 15.35100 +O 24.12100 1.96000 8.85000 +H 24.07100 1.91000 9.85000 +H 24.14100 2.92000 8.56000 +O 0.87100 14.56100 4.77000 +H 0.41100 15.28100 4.25000 +H 0.83100 14.76100 5.75000 +O 22.13100 18.01100 8.53000 +H 22.63100 17.15100 8.59000 +H 22.78100 18.78100 8.50000 +O 21.83100 9.43100 2.42000 +H 22.65100 9.82100 2.00000 +H 21.56100 8.61100 1.93000 +O 23.23100 12.66100 17.27100 +H 22.73100 12.67100 16.41100 +H 22.60100 12.48100 18.03100 +O 1.90100 17.76100 2.37000 +H 1.30100 17.14100 2.87000 +H 2.21100 17.32100 1.52000 +O 20.64100 2.85000 14.98100 +H 19.84100 3.45000 14.85100 +H 20.54100 2.36000 15.84100 +O 23.26100 17.43100 3.23000 +H 23.59100 17.82100 4.09000 +H 24.02100 17.36100 2.58000 +O 21.11100 17.85100 12.41100 +H 21.68100 17.20100 12.91100 +H 20.95100 17.52100 11.48100 +O 22.44100 7.00000 4.80000 +H 22.89100 6.10000 4.77000 +H 21.50100 6.89000 5.13000 +O 24.76100 1.22000 1.17000 +H 25.74100 1.00000 1.24000 +H 24.45100 1.05000 0.24000 +O 26.43100 2.64000 17.49100 +H 27.10100 2.03000 17.92100 +H 25.70100 2.83000 18.14100 +O 26.65100 14.02100 9.24000 +H 27.55100 14.16100 8.82000 +H 25.94100 14.04100 8.53000 +O 0.01100 5.03000 8.99000 +H -0.23900 4.94000 8.03000 +H 0.49100 4.21000 9.30000 +O 24.01100 0.64000 5.12000 +H 23.20100 0.65000 5.70000 +H 24.04100 1.47000 4.57000 +O 21.59100 0.35000 1.71000 +H 22.08100 1.19000 1.50000 +H 22.21100 -0.30000 2.16000 +O 0.14100 2.36000 4.80000 +H -0.33900 2.77000 4.02000 +H 1.13100 2.34000 4.61000 +O 23.21100 11.52100 7.41000 +H 22.50100 11.25100 8.06000 +H 22.95100 11.24100 6.48000 +O 19.40100 5.56000 13.86100 +H 20.32100 5.55000 13.45100 +H 19.34100 6.30000 14.53100 +O 24.23100 2.22000 11.47100 +H 24.61100 1.38000 11.84100 +H 23.35100 2.41000 11.91100 +O 27.28100 4.54000 6.42000 +H 26.96100 5.26000 5.80000 +H 27.52100 3.73000 5.89000 +O 23.50100 13.85100 1.74000 +H 22.63100 13.70100 2.21000 +H 23.33100 14.11100 0.79000 +O 23.89100 2.56000 3.28000 +H 24.16100 1.97000 2.53000 +H 23.89100 3.51000 2.97000 +O 27.77100 0.89000 14.02100 +H 28.02100 0.69000 13.07100 +H 28.49100 1.45000 14.44100 +O 22.72100 8.13000 12.51100 +H 23.58100 8.25000 13.01100 +H 22.30100 7.26000 12.78100 +O 19.26100 15.64100 13.31100 +H 18.80100 16.46100 12.97100 +H 20.24100 15.83100 13.40100 +O 22.29100 11.00100 5.01000 +H 22.22100 11.83100 4.45000 +H 22.33100 10.20100 4.41000 +O 24.28100 5.37000 8.65000 +H 24.40100 6.03000 7.91000 +H 24.74100 5.71000 9.48000 +O 22.16100 15.10100 13.29100 +H 21.95100 14.66100 12.42100 +H 23.13100 15.36100 13.32100 +O 22.64100 7.51000 15.98100 +H 23.32100 8.06000 15.51100 +H 23.04100 6.63000 16.25100 +O 23.66100 0.50000 17.40100 +H 23.24100 -0.07000 16.70100 +H 23.00100 1.19000 17.72100 +O 24.35100 8.70000 10.29100 +H 24.79100 9.59000 10.20100 +H 23.72100 8.70000 11.06100 +O 21.69100 2.13000 12.31100 +H 21.46100 2.50000 13.21100 +H 21.39100 1.18000 12.25100 +O 18.99100 13.10100 12.82100 +H 19.52100 12.61100 13.50100 +H 19.21100 14.08100 12.87100 +O 25.94100 6.34000 10.64100 +H 26.53100 6.08000 9.88100 +H 25.66100 7.30000 10.53100 +O 21.69100 0.63000 6.18000 +H 21.58100 1.57000 6.51000 +H 21.64100 0.00000 6.95000 +O 1.02100 7.66000 9.66100 +H 1.17100 7.87000 10.62100 +H 0.72100 6.71000 9.57100 +O 0.59100 15.73100 17.00100 +H 0.39100 16.17100 16.12100 +H 0.98100 14.82100 16.84100 +O 22.12100 8.98000 18.04100 +H 22.88100 9.42000 18.52100 +H 22.47100 8.51000 17.22100 +O 21.81100 8.10000 9.49100 +H 22.74100 8.46000 9.54100 +H 21.75100 7.25000 10.01100 +O 22.01100 5.09000 10.06100 +H 21.49100 4.26000 9.89100 +H 22.78100 5.14000 9.42100 +O 18.70100 13.26100 2.00000 +H 17.77100 13.47100 1.69000 +H 18.80100 12.27100 2.13000 +O 19.50100 18.01100 9.27000 +H 19.08100 17.15100 9.00000 +H 20.44100 18.04100 8.93000 +O 23.66100 15.68100 9.10000 +H 24.32100 16.42100 9.19000 +H 24.10100 14.89100 8.68000 +O 0.81100 7.96000 12.38100 +H 1.22100 7.64000 13.24100 +H -0.14900 7.69000 12.35100 +O 19.02100 5.44000 11.14100 +H 19.87100 5.11000 10.73100 +H 19.15100 5.59000 12.12100 +O 20.51100 5.20000 17.22100 +H 21.10100 4.80000 16.52100 +H 19.93100 5.91000 16.81100 +O 26.77100 5.72000 3.25000 +H 26.84100 4.83000 2.79000 +H 25.83100 6.06000 3.17000 +O 21.14100 15.64100 17.44100 +H 20.84100 16.21100 18.20100 +H 21.07100 14.67100 17.70100 +O 25.33100 4.64000 12.69100 +H 24.99100 3.75000 12.39100 +H 25.59100 5.18000 11.89100 +O 0.09100 16.78100 14.65100 +H -0.14900 16.60100 13.70100 +H 0.39100 17.72100 14.75100 +O 23.35100 5.00000 1.91000 +H 23.96100 5.80000 1.95000 +H 22.40100 5.31000 1.98000 +O 20.21100 11.37100 14.66100 +H 20.43100 10.76100 15.42100 +H 20.31100 10.88100 13.80100 +O 19.45100 12.58100 10.22100 +H 19.40100 12.57100 11.22100 +H 18.62100 12.17100 9.84100 +O 25.50100 16.62100 17.16100 +H 24.94100 17.43100 17.25100 +H 26.02100 16.66100 16.30100 +O 27.65100 0.86000 1.33000 +H 28.16100 0.87000 0.47000 +H 28.21100 0.44000 2.04000 +O 19.92100 17.94100 18.51100 +H 19.51100 17.20100 19.04100 +H 20.56100 18.45100 19.09100 +O 26.92100 12.73100 14.22100 +H 26.87100 13.06100 15.17100 +H 26.06100 12.92100 13.76100 +O 25.34100 16.16100 1.54000 +H 25.43100 16.26100 0.55000 +H 24.94100 15.27100 1.75000 +O 21.25100 3.26000 7.20000 +H 20.46100 3.77000 6.86000 +H 21.16100 3.11000 8.18000 +O 26.84100 10.02100 13.72100 +H 27.24100 10.01100 12.80100 +H 26.94100 10.94100 14.12100 +O 22.34100 12.88100 14.90100 +H 22.21100 13.81100 14.56100 +H 21.50100 12.36100 14.77100 +O 0.93100 2.36000 9.71100 +H 0.85100 2.00000 8.78100 +H 0.91100 1.60000 10.36100 +O 24.52100 14.87100 4.91000 +H 24.94100 14.29100 4.21000 +H 24.08100 15.66100 4.47000 +O 21.17100 13.48100 2.90000 +H 20.21100 13.49100 2.63000 +H 21.29100 14.01100 3.74000 +O 19.67100 10.13100 17.26100 +H 18.90100 9.80100 17.80100 +H 20.52100 9.83100 17.68100 +O 25.34100 2.03000 14.89100 +H 26.24100 1.87000 14.49100 +H 25.42100 2.08000 15.88100 +O 19.34100 1.66000 3.18000 +H 19.17100 2.49000 2.64000 +H 20.24100 1.29000 2.96000 +O 24.75100 8.42000 14.31100 +H 25.31100 9.23000 14.14100 +H 25.34100 7.62000 14.34100 +O 25.78100 5.65000 17.08100 +H 25.97100 6.30000 17.82100 +H 26.38100 4.85000 17.17100 +O 22.52100 17.41100 15.60100 +H 21.61100 17.82100 15.58100 +H 22.45100 16.47100 15.92100 +O 24.56100 7.45000 6.52000 +H 25.06100 8.30000 6.33000 +H 23.68100 7.47000 6.04000 +O 27.21100 13.74100 0.16000 +H 26.75100 13.89100 1.04000 +H 27.65100 14.59100 -0.14000 +O 25.23100 17.90100 9.53100 +H 24.77100 18.78100 9.40100 +H 26.22100 18.02100 9.46100 +O 27.21100 9.56100 8.61000 +H 27.75100 8.87100 9.09000 +H 26.89100 10.25100 9.27000 +O 20.83100 13.14100 18.44100 +H 20.18100 12.41100 18.23100 +H 20.87100 13.28100 19.42100 +O 19.41100 12.40100 6.53000 +H 19.40100 11.93100 7.41000 +H 20.23100 12.12100 6.02000 +O 25.34100 13.91100 16.24100 +H 24.56100 13.41100 16.62100 +H 25.31100 14.86100 16.55100 +O 22.90100 4.24000 5.20000 +H 23.20100 3.52000 4.58000 +H 22.51100 3.84000 6.03000 +O 21.79100 5.47000 12.80100 +H 22.17100 4.88000 13.52100 +H 22.19100 5.21000 11.92100 +O 26.74100 15.86100 6.87000 +H 27.06100 15.96100 5.93000 +H 25.95100 15.24100 6.89000 +O 0.80100 0.34000 11.54100 +H 0.17100 -0.38000 11.23100 +H 1.73100 -0.02000 11.54100 +O 26.32100 13.30100 3.01000 +H 25.86100 12.43100 3.18000 +H 27.23100 13.27100 3.42000 +O 24.80100 15.67100 12.84100 +H 24.75100 16.49100 13.41100 +H 25.69100 15.64100 12.39100 +O 19.01100 10.77100 3.00000 +H 20.00100 10.66100 2.91000 +H 18.61100 9.91100 3.32000 +O 27.37100 16.46100 3.37000 +H 26.60100 16.11100 2.83000 +H 27.05100 17.17100 3.99000 +O 20.92100 6.28000 19.75100 +H 19.99100 6.26000 20.12100 +H 20.93100 5.89000 18.83100 +O 20.87100 2.75000 0.75100 +H 21.22100 2.58000 1.67100 +H 19.99100 2.30000 0.63100 +O 18.81100 3.68000 25.09100 +H 17.99100 4.11000 25.48100 +H 18.53100 2.95000 24.46100 +O 26.30100 11.44100 1.02100 +H 25.52100 11.61100 1.62100 +H 26.64100 12.31100 0.66100 +O 25.47100 10.12100 25.27100 +H 26.16100 9.96100 25.97100 +H 24.74100 10.69100 25.65100 +O 20.93100 17.13100 23.45100 +H 21.27100 17.90100 23.99100 +H 21.37100 17.13100 22.55100 +O 20.92100 14.34100 24.00100 +H 20.66100 15.30100 24.00100 +H 20.21100 13.80100 24.45100 +O 21.02100 10.91100 27.48100 +H 21.16100 10.07100 28.00100 +H 20.47100 11.55100 28.03100 +O 24.68100 9.64100 19.85100 +H 24.75100 10.48100 19.31100 +H 25.14100 9.77100 20.73100 +O 19.84100 6.43000 24.25100 +H 19.39100 5.55000 24.42100 +H 19.83100 6.97000 25.09100 +O 18.89100 15.96100 19.79100 +H 18.70100 15.00100 20.00100 +H 18.56100 16.54100 20.54100 +O 24.75100 13.65100 25.88100 +H 24.26100 12.78100 25.97100 +H 24.52100 14.08100 25.01100 +O 26.71100 0.04000 23.64100 +H 27.11100 0.95000 23.55100 +H 25.71100 0.12000 23.70100 +O 24.60100 7.29000 21.32100 +H 24.84100 7.98000 20.64100 +H 23.82100 7.62000 21.86100 +O 1.20100 7.01000 22.91100 +H 1.46100 6.97000 23.87100 +H 0.35100 6.50000 22.77100 +O 19.38100 8.11000 26.51100 +H 20.37100 7.99000 26.60100 +H 19.14100 9.06000 26.72100 +O 0.87100 14.56100 23.39100 +H 0.41100 15.28100 22.87100 +H 0.83100 14.76100 24.37100 +O 22.13100 18.01100 27.15100 +H 22.63100 17.15100 27.21100 +H 22.78100 18.78100 27.12100 +O 21.83100 9.43100 21.04100 +H 22.65100 9.82100 20.62100 +H 21.56100 8.61100 20.55100 +O 1.90100 17.76100 20.99100 +H 1.30100 17.14100 21.49100 +H 2.21100 17.32100 20.14100 +O 23.26100 17.43100 21.85100 +H 23.59100 17.82100 22.71100 +H 24.02100 17.36100 21.20100 +O 22.44100 7.00000 23.42100 +H 22.89100 6.10000 23.39100 +H 21.50100 6.89000 23.75100 +O 24.76100 1.22000 19.79100 +H 25.74100 1.00000 19.86100 +H 24.45100 1.05000 18.86100 +O 0.01100 5.03000 27.61100 +H -0.23900 4.94000 26.65100 +H 0.49100 4.21000 27.92100 +O 24.01100 0.64000 23.74100 +H 23.20100 0.65000 24.32100 +H 24.04100 1.47000 23.19100 +O 21.59100 0.35000 20.33100 +H 22.08100 1.19000 20.12100 +H 22.21100 -0.30000 20.78100 +O 0.14100 2.36000 23.42100 +H -0.33900 2.77000 22.64100 +H 1.13100 2.34000 23.23100 +O 23.21100 11.52100 26.03100 +H 22.50100 11.25100 26.68100 +H 22.95100 11.24100 25.10100 +O 27.28100 4.54000 25.04100 +H 26.96100 5.26000 24.42100 +H 27.52100 3.73000 24.51100 +O 23.50100 13.85100 20.36100 +H 22.63100 13.70100 20.83100 +H 23.33100 14.11100 19.41100 +O 23.89100 2.56000 21.90100 +H 24.16100 1.97000 21.15100 +H 23.89100 3.51000 21.59100 +O 22.29100 11.00100 23.63100 +H 22.22100 11.83100 23.07100 +H 22.33100 10.20100 23.03100 +O 24.28100 5.37000 27.27100 +H 24.40100 6.03000 26.53100 +H 24.74100 5.71000 28.10100 +O 21.69100 0.63000 24.80100 +H 21.58100 1.57000 25.13100 +H 21.64100 0.00000 25.57100 +O 1.02100 7.66000 0.45100 +H 1.17100 7.87000 1.41100 +H 0.72100 6.71000 0.36100 +O 18.70100 13.26100 20.62100 +H 17.77100 13.47100 20.31100 +H 18.80100 12.27100 20.75100 +O 19.50100 18.01100 0.06100 +H 19.08100 17.15100 -0.20900 +H 20.44100 18.04100 -0.27900 +O 26.77100 5.72000 21.87100 +H 26.84100 4.83000 21.41100 +H 25.83100 6.06000 21.79100 +O 23.35100 5.00000 20.53100 +H 23.96100 5.80000 20.57100 +H 22.40100 5.31000 20.60100 +O 25.34100 16.16100 20.16100 +H 25.43100 16.26100 19.17100 +H 24.94100 15.27100 20.37100 +O 21.25100 3.26000 25.82100 +H 20.46100 3.77000 25.48100 +H 21.16100 3.11000 26.80100 +O 24.52100 14.87100 23.53100 +H 24.94100 14.29100 22.83100 +H 24.08100 15.66100 23.09100 +O 21.17100 13.48100 21.52100 +H 20.21100 13.49100 21.25100 +H 21.29100 14.01100 22.36100 +O 19.37100 3.45000 18.95100 +H 18.45100 3.17000 18.66100 +H 19.68100 4.22000 18.39100 +O 19.34100 1.66000 21.80100 +H 19.17100 2.49000 21.26100 +H 20.24100 1.29000 21.58100 +O 24.56100 7.45000 25.14100 +H 25.06100 8.30000 24.95100 +H 23.68100 7.47000 24.66100 +O 27.21100 13.74100 18.78100 +H 26.75100 13.89100 19.66100 +H 27.65100 14.59100 18.48100 +O 25.23100 17.90100 0.32100 +H 24.77100 18.78100 0.19100 +H 26.22100 18.02100 0.25100 +O 27.21100 9.56100 27.23100 +H 27.75100 8.87100 27.71100 +H 26.89100 10.25100 27.89100 +O 1.62100 9.84100 19.49100 +H 1.39100 10.37100 18.67100 +H 2.43100 9.28100 19.30100 +O 19.41100 12.40100 25.15100 +H 19.40100 11.93100 26.03100 +H 20.23100 12.12100 24.64100 +O 22.90100 4.24000 23.82100 +H 23.20100 3.52000 23.20100 +H 22.51100 3.84000 24.65100 +O 26.74100 15.86100 25.49100 +H 27.06100 15.96100 24.55100 +H 25.95100 15.24100 25.51100 +O 26.32100 13.30100 21.63100 +H 25.86100 12.43100 21.80100 +H 27.23100 13.27100 22.04100 +O 19.01100 10.77100 21.62100 +H 20.00100 10.66100 21.53100 +H 18.61100 9.91100 21.94100 +O 27.37100 16.46100 21.99100 +H 26.60100 16.11100 21.45100 +H 27.05100 17.17100 22.61100 +O 20.92100 24.90100 1.13000 +H 19.99100 24.88100 1.50000 +H 20.93100 24.51100 0.21000 +O 20.87100 21.37100 9.96100 +H 21.22100 21.20100 10.88100 +H 19.99100 20.92100 9.84100 +O 18.81100 22.30100 6.47000 +H 17.99100 22.73100 6.86000 +H 18.53100 21.57100 5.84000 +O 27.12100 26.60100 18.23100 +H 27.08100 27.36100 18.88100 +H 27.34100 26.96100 17.32100 +O 25.47100 0.91100 6.65000 +H 26.16100 0.75100 7.35000 +H 24.74100 1.48100 7.03000 +O 23.00100 22.54100 14.99100 +H 23.82100 21.98100 15.08100 +H 22.19100 21.96100 15.03100 +O 19.84100 25.05100 5.63000 +H 19.39100 24.17100 5.80000 +H 19.83100 25.59100 6.47000 +O 19.75100 25.99100 15.97100 +H 20.63100 25.86100 16.42100 +H 19.62100 26.96100 15.75100 +O 26.71100 18.66100 5.02000 +H 27.11100 19.57100 4.93000 +H 25.71100 18.74100 5.08000 +O 21.79100 21.13100 18.01100 +H 22.50100 21.84100 18.07100 +H 20.91100 21.52100 18.29100 +O 24.60100 25.91100 2.70000 +H 24.84100 26.60100 2.02000 +H 23.82100 26.24100 3.24000 +O 26.69100 24.67100 14.65100 +H 26.22100 24.64100 15.54100 +H 26.18100 24.12100 13.99100 +O 1.20100 25.63100 4.29000 +H 1.46100 25.59100 5.25000 +H 0.35100 25.12100 4.15000 +O 19.38100 26.73100 7.89000 +H 20.37100 26.61100 7.98000 +H 19.14100 27.68100 8.10000 +O 24.12100 20.58100 8.85000 +H 24.07100 20.53100 9.85000 +H 24.14100 21.54100 8.56000 +O 20.64100 21.47100 14.98100 +H 19.84100 22.07100 14.85100 +H 20.54100 20.98100 15.84100 +O 22.44100 25.62100 4.80000 +H 22.89100 24.72100 4.77000 +H 21.50100 25.51100 5.13000 +O 26.43100 21.26100 17.49100 +H 27.10100 20.65100 17.92100 +H 25.70100 21.45100 18.14100 +O 0.01100 23.65100 8.99000 +H -0.23900 23.56100 8.03000 +H 0.49100 22.83100 9.30000 +O 24.01100 19.26100 5.12000 +H 23.20100 19.27100 5.70000 +H 24.04100 20.09100 4.57000 +O 21.59100 18.97100 1.71000 +H 22.08100 19.81100 1.50000 +H 22.21100 18.32100 2.16000 +O 0.14100 20.98100 4.80000 +H -0.33900 21.39100 4.02000 +H 1.13100 20.96100 4.61000 +O 23.21100 2.31100 7.41000 +H 22.50100 2.04100 8.06000 +H 22.95100 2.03100 6.48000 +O 19.40100 24.18100 13.86100 +H 20.32100 24.17100 13.45100 +H 19.34100 24.92100 14.53100 +O 24.23100 20.84100 11.47100 +H 24.61100 20.00100 11.84100 +H 23.35100 21.03100 11.91100 +O 27.28100 23.16100 6.42000 +H 26.96100 23.88100 5.80000 +H 27.52100 22.35100 5.89000 +O 0.96100 19.01100 7.53000 +H 0.24100 19.06100 6.84000 +H 0.72100 18.32100 8.22000 +O 23.89100 21.18100 3.28000 +H 24.16100 20.59100 2.53000 +H 23.89100 22.13100 2.97000 +O 27.77100 19.51100 14.02100 +H 28.02100 19.31100 13.07100 +H 28.49100 20.07100 14.44100 +O 22.72100 26.75100 12.51100 +H 23.58100 26.87100 13.01100 +H 22.30100 25.88100 12.78100 +O 24.28100 23.99100 8.65000 +H 24.40100 24.65100 7.91000 +H 24.74100 24.33100 9.48000 +O 22.64100 26.13100 15.98100 +H 23.32100 26.68100 15.51100 +H 23.04100 25.25100 16.25100 +O 0.92100 19.67100 17.70100 +H 0.98100 20.52100 17.18100 +H 1.24100 18.91100 17.13100 +O 23.66100 19.12100 17.40100 +H 23.24100 18.55100 16.70100 +H 23.00100 19.81100 17.72100 +O 24.35100 27.32100 10.29100 +H 24.79100 28.21100 10.20100 +H 23.72100 27.32100 11.06100 +O 21.69100 20.75100 12.31100 +H 21.46100 21.12100 13.21100 +H 21.39100 19.80100 12.25100 +O 25.94100 24.96100 10.64100 +H 26.53100 24.70100 9.88100 +H 25.66100 25.92100 10.53100 +O 21.69100 19.25100 6.18000 +H 21.58100 20.19100 6.51000 +H 21.64100 18.62100 6.95000 +O 1.02100 26.28100 9.66100 +H 1.17100 26.49100 10.62100 +H 0.72100 25.33100 9.57100 +O 21.81100 26.72100 9.49100 +H 22.74100 27.08100 9.54100 +H 21.75100 25.87100 10.01100 +O 22.01100 23.71100 10.06100 +H 21.49100 22.88100 9.89100 +H 22.78100 23.76100 9.42100 +O 23.73100 22.77100 18.08100 +H 23.55100 23.22100 18.96100 +H 24.15100 23.42100 17.45100 +O 0.81100 26.58100 12.38100 +H 1.22100 26.26100 13.24100 +H -0.14900 26.31100 12.35100 +O 19.02100 24.06100 11.14100 +H 19.87100 23.73100 10.73100 +H 19.15100 24.21100 12.12100 +O 20.51100 23.82100 17.22100 +H 21.10100 23.42100 16.52100 +H 19.93100 24.53100 16.81100 +O 26.77100 24.34100 3.25000 +H 26.84100 23.45100 2.79000 +H 25.83100 24.68100 3.17000 +O 25.33100 23.26100 12.69100 +H 24.99100 22.37100 12.39100 +H 25.59100 23.80100 11.89100 +O 23.35100 23.62100 1.91000 +H 23.96100 24.42100 1.95000 +H 22.40100 23.93100 1.98000 +O 27.65100 19.48100 1.33000 +H 28.16100 19.49100 0.47000 +H 28.21100 19.06100 2.04000 +O 21.25100 21.88100 7.20000 +H 20.46100 22.39100 6.86000 +H 21.16100 21.73100 8.18000 +O 1.18100 1.77100 6.96000 +H 0.48100 1.30100 7.50000 +H 1.77100 1.09100 6.53000 +O 0.93100 20.98100 9.71100 +H 0.85100 20.62100 8.78100 +H 0.91100 20.22100 10.36100 +O 19.67100 0.92100 17.26100 +H 18.90100 0.59100 17.80100 +H 20.52100 0.62100 17.68100 +O 25.34100 20.65100 14.89100 +H 26.24100 20.49100 14.49100 +H 25.42100 20.70100 15.88100 +O 19.34100 20.28100 3.18000 +H 19.17100 21.11100 2.64000 +H 20.24100 19.91100 2.96000 +O 24.75100 27.04100 14.31100 +H 25.31100 27.85100 14.14100 +H 25.34100 26.24100 14.34100 +O 25.78100 24.27100 17.08100 +H 25.97100 24.92100 17.82100 +H 26.38100 23.47100 17.17100 +O 24.56100 26.07100 6.52000 +H 25.06100 26.92100 6.33000 +H 23.68100 26.09100 6.04000 +O 27.21100 0.35100 8.61000 +H 27.75100 -0.33900 9.09000 +H 26.89100 1.04100 9.27000 +O 22.90100 22.86100 5.20000 +H 23.20100 22.14100 4.58000 +H 22.51100 22.46100 6.03000 +O 21.79100 24.09100 12.80100 +H 22.17100 23.50100 13.52100 +H 22.19100 23.83100 11.92100 +O 0.80100 18.96100 11.54100 +H 0.17100 18.24100 11.23100 +H 1.73100 18.60100 11.54100 +O 20.92100 24.90100 19.75100 +H 19.99100 24.88100 20.12100 +H 20.93100 24.51100 18.83100 +O 20.87100 21.37100 0.75100 +H 21.22100 21.20100 1.67100 +H 19.99100 20.92100 0.63100 +O 18.81100 22.30100 25.09100 +H 17.99100 22.73100 25.48100 +H 18.53100 21.57100 24.46100 +O 25.47100 0.91100 25.27100 +H 26.16100 0.75100 25.97100 +H 24.74100 1.48100 25.65100 +O 19.84100 25.05100 24.25100 +H 19.39100 24.17100 24.42100 +H 19.83100 25.59100 25.09100 +O 26.71100 18.66100 23.64100 +H 27.11100 19.57100 23.55100 +H 25.71100 18.74100 23.70100 +O 24.60100 25.91100 21.32100 +H 24.84100 26.60100 20.64100 +H 23.82100 26.24100 21.86100 +O 1.20100 25.63100 22.91100 +H 1.46100 25.59100 23.87100 +H 0.35100 25.12100 22.77100 +O 19.38100 26.73100 26.51100 +H 20.37100 26.61100 26.60100 +H 19.14100 27.68100 26.72100 +O 24.12100 20.58100 27.47100 +H 24.07100 20.53100 28.47100 +H 24.14100 21.54100 27.18100 +O 22.44100 25.62100 23.42100 +H 22.89100 24.72100 23.39100 +H 21.50100 25.51100 23.75100 +O 24.76100 19.84100 19.79100 +H 25.74100 19.62100 19.86100 +H 24.45100 19.67100 18.86100 +O 0.01100 23.65100 27.61100 +H -0.23900 23.56100 26.65100 +H 0.49100 22.83100 27.92100 +O 24.01100 19.26100 23.74100 +H 23.20100 19.27100 24.32100 +H 24.04100 20.09100 23.19100 +O 21.59100 18.97100 20.33100 +H 22.08100 19.81100 20.12100 +H 22.21100 18.32100 20.78100 +O 0.14100 20.98100 23.42100 +H -0.33900 21.39100 22.64100 +H 1.13100 20.96100 23.23100 +O 23.21100 2.31100 26.03100 +H 22.50100 2.04100 26.68100 +H 22.95100 2.03100 25.10100 +O 27.28100 23.16100 25.04100 +H 26.96100 23.88100 24.42100 +H 27.52100 22.35100 24.51100 +O 0.96100 19.01100 26.15100 +H 0.24100 19.06100 25.46100 +H 0.72100 18.32100 26.84100 +O 23.89100 21.18100 21.90100 +H 24.16100 20.59100 21.15100 +H 23.89100 22.13100 21.59100 +O 24.28100 23.99100 27.27100 +H 24.40100 24.65100 26.53100 +H 24.74100 24.33100 28.10100 +O 21.69100 19.25100 24.80100 +H 21.58100 20.19100 25.13100 +H 21.64100 18.62100 25.57100 +O 1.02100 26.28100 0.45100 +H 1.17100 26.49100 1.41100 +H 0.72100 25.33100 0.36100 +O 26.77100 24.34100 21.87100 +H 26.84100 23.45100 21.41100 +H 25.83100 24.68100 21.79100 +O 23.35100 23.62100 20.53100 +H 23.96100 24.42100 20.57100 +H 22.40100 23.93100 20.60100 +O 21.25100 21.88100 25.82100 +H 20.46100 22.39100 25.48100 +H 21.16100 21.73100 26.80100 +O 27.78100 27.72100 21.53100 +H 28.41100 28.10100 20.85100 +H 28.18100 26.89100 21.92100 +O 1.18100 1.77100 25.58100 +H 0.48100 1.30100 26.12100 +H 1.77100 1.09100 25.15100 +O 19.37100 22.07100 18.95100 +H 18.45100 21.79100 18.66100 +H 19.68100 22.84100 18.39100 +O 19.34100 20.28100 21.80100 +H 19.17100 21.11100 21.26100 +H 20.24100 19.91100 21.58100 +O 24.56100 26.07100 25.14100 +H 25.06100 26.92100 24.95100 +H 23.68100 26.09100 24.66100 +O 1.62100 0.63100 19.49100 +H 1.39100 1.16100 18.67100 +H 2.43100 0.07100 19.30100 +O 22.90100 22.86100 23.82100 +H 23.20100 22.14100 23.20100 +H 22.51100 22.46100 24.65100 diff --git a/examples/gpmdk/run/water/coords_300.dat b/examples/gpmdk/run/water/coords_300.dat new file mode 100644 index 00000000..6eec1584 --- /dev/null +++ b/examples/gpmdk/run/water/coords_300.dat @@ -0,0 +1,304 @@ + 300 + 14.40500 0.00000 0.00000 + 0.00000 14.40500 0.00000 + 0.00000 0.00000 14.40500 +H 2.44100 0.20600 1.00300 +O 2.28300 0.06800 0.04900 +H 1.79100 13.63200 0.01400 +H 0.50700 14.25000 6.56400 +O 0.89500 0.74100 6.43800 +H 0.12600 1.30400 6.65900 +H 6.07100 4.56800 13.85400 +O 6.37600 5.36600 14.36300 +H 6.85400 4.94600 0.72800 +H 9.13800 4.82900 5.13900 +O 9.29500 4.91700 4.17100 +H 10.26100 4.85100 4.00400 +H 8.42200 9.67800 11.47900 +O 7.63900 10.22400 11.24900 +H 6.98500 10.03000 11.95900 +H 6.55500 7.29100 3.44600 +O 7.45600 7.68200 3.53200 +H 7.49900 7.84300 4.50800 +H 4.03200 0.78800 4.74000 +O 4.75200 0.59500 5.39200 +H 4.93100 14.05400 5.27900 +H 1.04800 12.26800 2.34300 +O 1.88300 12.29500 1.83400 +H 1.86500 11.48000 1.30400 +H 2.74800 5.73400 0.76500 +O 3.46700 5.76700 0.11100 +H 3.92400 4.91000 0.17200 +H 14.40100 8.01500 4.04100 +O 0.76200 8.54100 3.68700 +H 1.00500 8.03800 2.88300 +H 7.57100 5.54500 7.83500 +O 7.65800 5.22700 6.90000 +H 7.10000 4.43000 6.98000 +H 12.14000 7.26200 7.79000 +O 13.06300 7.56900 7.96500 +H 13.60300 6.79600 7.66600 +H 5.93400 8.72200 10.97800 +O 5.13500 8.66700 11.53300 +H 4.54900 8.03000 11.07400 +H 7.15000 0.49500 4.78800 +O 8.03300 0.64100 5.18500 +H 8.68400 0.54600 4.46200 +H 1.35800 11.09600 7.49700 +O 1.66800 11.00200 6.56900 +H 1.84200 10.05600 6.42900 +H 2.32300 0.33300 7.72900 +O 3.22700 14.38400 7.87200 +H 3.04600 13.58400 8.41900 +H 9.01000 4.35900 10.22700 +O 9.93700 4.62600 10.06700 +H 10.04200 4.45800 9.11500 +H 2.06900 2.69100 13.04500 +O 2.59600 2.79900 13.86800 +H 2.55700 1.89200 14.23700 +H 5.72700 4.79800 4.39600 +O 6.51700 5.06700 3.85800 +H 7.25200 4.96500 4.49800 +H 2.69800 5.51700 7.46500 +O 3.02200 6.39100 7.77500 +H 3.46900 6.21700 8.63900 +H 0.76700 4.55300 4.59100 +O 0.38900 3.83000 4.05200 +H 14.34500 4.32900 3.33800 +H 12.18900 4.33400 10.12700 +O 12.65400 4.89700 10.77400 +H 12.54400 4.41900 11.61900 +H 8.10500 13.66900 2.56500 +O 8.43200 13.12200 3.29900 +H 7.81600 12.42300 3.59800 +H 13.16500 9.90100 12.67200 +O 14.15900 9.90100 12.39300 +H 14.27000 9.05600 11.92800 +H 7.84000 13.93900 9.33300 +O 7.92300 14.23400 10.25800 +H 7.40000 13.58600 10.76200 +H 6.28400 0.41000 8.01800 +O 5.69000 1.17500 8.11800 +H 4.84100 0.82000 7.71700 +H 1.94900 12.85000 10.69600 +O 2.01500 12.54100 9.76500 +H 1.28400 11.89400 9.72200 +H 14.31000 7.21700 10.10700 +O 0.47300 7.78600 10.64900 +H 1.34500 7.36400 10.58100 +H 1.91600 0.79200 3.31800 +O 2.78500 0.62900 2.89000 +H 3.06600 14.17500 3.29700 +H 10.53100 2.50700 2.36700 +O 11.36600 2.91800 2.69700 +H 11.56000 2.47700 3.54000 +H 3.27600 5.45800 11.53400 +O 4.15400 5.85000 11.33800 +H 4.43800 6.12500 12.23200 +H 7.00100 8.93100 9.33700 +O 7.39500 8.40400 8.62600 +H 7.02100 7.49300 8.72600 +H 5.75200 5.35800 9.37400 +O 6.56800 5.90700 9.32100 +H 6.88200 5.97000 10.27200 +H 10.40400 12.64500 6.10700 +O 10.85000 13.13700 5.38900 +H 10.99800 12.46100 4.68700 +H 10.61900 10.58000 13.77000 +O 11.59400 10.52100 13.96100 +H 11.70300 9.55700 14.13200 +H 4.10700 1.50300 1.76500 +O 4.94200 1.40000 1.27600 +H 4.85000 0.48400 0.94400 +H 7.95300 12.37300 7.74500 +O 8.51700 12.17400 8.51900 +H 8.04200 11.47200 8.99800 +H 8.65200 3.68700 0.28100 +O 9.36200 3.39000 14.09900 +H 8.93600 2.79700 13.48000 +H 0.50300 8.64900 13.75200 +O 1.28100 8.48500 14.34500 +H 1.02900 7.68600 0.48100 +H 4.43000 12.76400 9.75100 +O 4.82500 13.52700 10.22600 +H 5.21400 14.07100 9.51700 +H 2.58000 9.53500 9.06300 +O 3.17400 8.77600 8.89400 +H 3.46100 8.85100 7.96600 +H 4.11900 12.68100 13.25000 +O 4.22500 12.06400 12.48900 +H 3.84800 11.22300 12.85000 +H 3.71200 13.00100 0.99200 +O 4.62800 12.95800 0.64700 +H 4.98900 12.13800 1.10600 +H 7.56200 9.50400 3.04100 +O 8.04500 10.34400 3.25300 +H 9.00100 10.12000 3.12500 +H 6.65100 11.71200 2.12800 +O 5.77600 11.41900 2.45900 +H 5.84800 10.44500 2.46400 +H 12.43900 6.66900 1.83900 +O 11.90900 6.01200 2.32300 +H 12.22900 5.14600 2.00300 +H 0.01500 5.86600 1.71600 +O 0.86300 6.29800 1.95400 +H 1.47200 5.58300 2.19800 +H 11.93100 12.50200 8.47200 +O 11.09900 12.76900 8.03300 +H 10.40400 12.33000 8.55900 +H 4.18500 7.37800 6.80400 +O 4.58800 8.16400 6.36200 +H 4.06900 8.26300 5.54600 +H 13.90600 2.01700 8.61300 +O 0.42000 2.40200 8.55700 +H 0.72200 2.32600 9.49200 +H 4.77800 3.30700 11.98200 +O 5.22100 2.98700 12.79900 +H 4.50400 3.06700 13.48000 +H 13.34400 13.91800 3.35700 +O 14.12600 14.28700 3.81400 +H 0.32500 13.51800 3.91800 +H 9.03700 6.80200 13.69500 +O 9.60000 7.46700 13.23300 +H 9.42900 7.30100 12.27400 +H 0.50800 11.30700 5.18600 +O 0.38100 11.38200 4.20800 +H 0.87200 10.59300 3.84800 +H 12.97800 3.42400 1.81800 +O 13.70400 3.93900 1.38500 +H 13.84500 3.49900 0.52700 +H 12.19600 6.94000 4.04700 +O 13.00400 7.21100 4.53000 +H 12.67700 7.39900 5.42900 +H 10.42200 0.27300 11.89100 +O 10.91400 0.38200 11.04600 +H 10.21900 0.31100 10.36600 +H 7.67500 2.21900 8.04800 +O 8.31300 2.58900 7.40300 +H 8.25800 1.94000 6.64500 +H 10.07400 10.19500 6.81700 +O 9.89800 10.16400 5.86100 +H 10.36600 9.35900 5.57300 +H 12.13400 1.07200 9.93600 +O 12.23300 1.27900 8.95600 +H 12.27200 0.42600 8.41600 +H 4.05900 7.71300 1.79200 +O 4.71500 7.10600 2.16000 +H 4.64800 6.38100 1.50800 +H 13.87700 10.61100 8.87100 +O 0.38900 10.36300 9.16200 +H 0.24000 9.46200 9.49100 +H 2.96700 9.01700 14.01200 +O 3.78500 9.58100 13.91400 +H 4.27700 9.04900 13.24700 +H 10.08500 14.00000 2.85900 +O 10.87300 14.37900 2.55900 +H 11.33800 0.39000 3.32600 +H 11.30900 7.53200 14.04400 +O 12.22900 7.70000 14.38900 +H 12.79700 7.04500 13.91200 +H 6.46100 13.93100 0.93500 +O 7.39600 13.67800 0.82500 +H 7.40600 13.36300 14.29800 +H 12.03000 1.44800 12.36400 +O 12.79400 1.37100 12.95000 +H 12.48500 0.71800 13.59900 +H 2.12200 3.25800 7.72200 +O 2.41000 3.77400 6.94200 +H 1.76400 3.47600 6.27600 +H 4.22800 3.98700 6.00400 +O 5.09600 3.72200 5.62800 +H 4.96000 2.80100 5.32800 +H 6.04700 9.52000 6.08700 +O 6.21900 10.28600 5.51600 +H 7.18900 10.39800 5.52700 +H 8.77800 12.11800 0.33800 +O 8.80000 11.14300 0.34500 +H 8.72500 10.88600 1.28400 +H 10.10700 3.02100 7.58500 +O 10.98000 3.50500 7.61600 +H 11.55900 2.82600 8.08000 +H 3.81500 0.37900 11.05900 +O 3.29900 1.19700 10.81100 +H 3.13700 1.04300 9.85600 +H 13.45000 5.79100 12.68600 +O 14.08500 6.44400 13.02200 +H 0.43100 6.35200 12.39500 +H 3.42700 12.06000 4.47100 +O 3.08300 12.95500 4.64600 +H 2.46000 12.84000 5.38600 +H 13.38900 10.00500 5.93700 +O 13.06200 10.09100 6.85900 +H 13.13100 9.17900 7.24100 +H 9.11400 7.98200 9.50300 +O 9.51600 7.58100 10.31200 +H 9.61800 6.62100 10.11700 +H 4.86000 3.20900 9.65100 +O 4.91300 3.71100 8.81700 +H 5.20200 3.03100 8.17800 +H 11.40400 9.36900 2.31000 +O 10.88500 9.86400 2.97000 +H 11.37800 9.78200 3.81300 +H 7.23000 3.78400 2.58200 +O 7.22100 3.20900 1.77800 +H 6.52200 2.53200 1.91800 +H 9.77400 8.20900 3.61000 +O 10.15200 7.57100 4.25100 +H 9.45400 6.89400 4.34800 +H 12.73400 13.13300 6.28600 +O 13.58500 13.10700 6.77900 +H 13.71100 12.13200 6.92500 +H 10.94300 4.25600 13.77000 +O 11.64400 4.91500 13.57600 +H 11.18700 5.75300 13.76000 +H 13.17500 4.66400 6.75600 +O 13.93700 4.92600 7.30800 +H 0.00800 4.08900 7.49900 +H 9.23200 0.02700 14.11100 +O 10.09800 14.15000 13.77100 +H 10.64300 14.14700 0.17500 +H 11.54100 0.53900 5.36700 +O 12.05300 1.30000 4.99700 +H 12.97700 0.98000 4.91300 +H 10.54600 11.50500 11.84100 +O 11.22400 11.83900 11.21900 +H 11.17800 12.81700 11.31600 +H 9.91800 6.03300 6.35500 +O 10.79300 6.15800 6.76000 +H 10.87500 5.36000 7.33500 +H 6.86300 2.48500 10.77100 +O 7.80800 2.52200 11.06600 +H 8.10000 1.59000 10.91800 +H 5.25000 12.79800 6.97500 +O 6.22400 12.78300 6.96300 +H 6.45400 12.10200 6.30800 +H 1.87300 3.69600 2.82600 +O 2.60200 3.96600 2.23300 +H 3.06200 3.14900 1.99600 +H 12.48600 12.31900 0.01700 +O 12.77300 13.11700 0.52100 +H 11.96200 13.45700 0.95200 +H 6.19400 12.38600 12.57000 +O 7.09800 12.71300 12.49900 +H 7.62200 12.00600 12.02800 +H 0.51200 1.65100 12.21400 +O 0.95800 2.07500 11.45200 +H 1.85400 1.62600 11.33700 +H 0.03200 13.04100 12.91800 +O 0.43200 12.61800 12.10900 +H 0.18700 11.64900 12.11500 +H 9.77600 5.76100 1.70900 +O 8.80000 5.83300 1.60300 +H 8.52300 6.45400 2.32500 +H 7.31400 5.72700 12.84400 +O 7.63500 5.59900 11.90600 +H 7.84700 4.59300 11.87700 +H 5.53800 9.45400 0.42400 +O 6.40700 9.29200 0.86200 +H 6.27900 8.38500 1.19900 +H 4.40000 10.34500 9.33100 +O 4.49200 11.09000 8.70400 +H 4.12200 10.74300 7.87400 +H 0.08700 4.80700 10.70500 +O 0.94000 4.68900 11.19200 +H 0.90200 3.73400 11.48000 diff --git a/examples/gpmdk/run/water/get_energy.py b/examples/gpmdk/run/water/get_energy.py new file mode 100755 index 00000000..2da423a3 --- /dev/null +++ b/examples/gpmdk/run/water/get_energy.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import argparse +import re + +energy_re = re.compile("Energy Total.*=\s+([0-9-.]+)") + +parser = argparse.ArgumentParser() +parser.add_argument("OUT", + help="The output") +options = parser.parse_args() + +with open(options.OUT) as fd: + for line in fd: + result = energy_re.search(line) + if result: + print(result.group(1)) diff --git a/examples/gpmdk/run/water/gpmd_input.in b/examples/gpmdk/run/water/gpmd_input.in new file mode 100644 index 00000000..2ff3af1d --- /dev/null +++ b/examples/gpmdk/run/water/gpmd_input.in @@ -0,0 +1,153 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 10 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + CoordsFile= "CH4.pdb" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.1 + #TimeStep= 0.00 + MDSteps= 200000 + ParamPath= "../sulfurTBparam" + NlistEach= 1 + MuCalcType= FromParts + EFermi= -0.0 + kBT= 0.025 + Entropy= T + DoKernel= F +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + #GraphElement= Orbital + #PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + #PartitionType= METIS + PartitionType= Block + NodesPerPart= 0 + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 4 + PartitionCount= 1 + #PartitionCount= 512 + #PartitionCount= 16 + #PartitionCount= 2 + #PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 32 + #PartitionCount= 64 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= -1 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + BuildAlways= F + RankNUpdate= 3 + KernelMixing= F + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 1 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + WriteCoordsEach= 1 + #TrajectoryFormat= XYZ + InitialTemperature= 300 + #HtoD= T + MinimizationSteps= 100 + LangevinDynamics= T + LangevinMethod= Siva + UseSMD= F + #For polyaniline + #SMDAtomIndex1= 6 + #SMDAtomIndex2= 7 + + SMDForceConstant= -8.0 + SMDR0= 0.001 + ComputeCurrents= T + CurrentThreshold= 0.0 +} diff --git a/examples/gpmdk/run/water/input.in b/examples/gpmdk/run/water/input.in new file mode 100644 index 00000000..014a48ce --- /dev/null +++ b/examples/gpmdk/run/water/input.in @@ -0,0 +1,182 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + #StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.2 + #TimeStep= 0.00 + MDSteps= 1000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + CoordsFile= coords_300.dat + #CoordsFile= coords_2088.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb + NlistEach= 1 + #BMLDistributionType= Distributed + MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn + EFermi= -0.0 + #kBT= 0.04308695 + kBT= 0.025 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + #PartitionType= METIS + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 256 + #PartitionCount= 512 + #PartitionCount= 16 + PartitionCount= 1 + #PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 32 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= -1 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks + BuildAlways= F + RankNUpdate= 4 + KernelMixing= T + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 1 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + WriteCoordsEach= 10 + LangevinMethod= Siva + LangevinDynamics= T + LangevinGamma= 0.01 + InitialTemperature= 300.0 +} diff --git a/examples/gpmdk/run/water/input_single.in b/examples/gpmdk/run/water/input_single.in new file mode 100644 index 00000000..967142af --- /dev/null +++ b/examples/gpmdk/run/water/input_single.in @@ -0,0 +1,182 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + #StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.2 + #TimeStep= 0.00 + MDSteps= 10000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + CoordsFile= coords_300.dat + #CoordsFile= coords_2088.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb + NlistEach= 1 + #BMLDistributionType= Distributed + MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn + EFermi= -0.0 + #kBT= 0.04308695 + kBT= 0.025 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + #PartitionType= METIS + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 256 + #PartitionCount= 512 + #PartitionCount= 16 + PartitionCount= 1 + #PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 32 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= -1 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks + BuildAlways= F + RankNUpdate= 4 + KernelMixing= T + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 1 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + WriteCoordsEach= 10 + LangevinMethod= Siva + LangevinDynamics= T + LangevinGamma= 0.01 + InitialTemperature= 300.0 +} diff --git a/examples/gpmdk/run/water/job b/examples/gpmdk/run/water/job new file mode 100755 index 00000000..1ecc0cf0 --- /dev/null +++ b/examples/gpmdk/run/water/job @@ -0,0 +1,19 @@ +#!/bin/bash + +#SBATCH -N32 #Number of cores and nodes +##SBATCH -N1 #Number of cores and nodes +#SBATCH --ntasks-per-node=72 +#SBATCH -p scaling --time=8:00:00 +# OMP_NUM_THREADS=36 controls the number of threads your application use +## This variable cat be set by the following command : + +source $HOME/BESGraph/qmd-progress/examples/gpmdcov/vars + +## Run command +export OMP_NUM_THREADS=36 +RUN="$HOME/BESGraph/qmd-progress/build/gpmdcov" +#OMP_PROC_BIND=spread OMP_NUM_THREADS=4 mpirun -n 128 --map-by ppr:8:node:pe=4 $RUN input.in > out +#OMP_PROC_BIND=spread OMP_NUM_THREADS=36 mpirun -n 1 --map-by ppr:4:node:pe=36 $RUN input.in > out +#OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 64 --map-by ppr:4:node:pe=9 $RUN input.in > out +OMP_PROC_BIND=spread OMP_NUM_THREADS=9 mpirun -n 16 --map-by ppr:4:node:pe=9 $RUN input.in > out + diff --git a/examples/gpmdk/run/water/latte.in b/examples/gpmdk/run/water/latte.in new file mode 100644 index 00000000..59947cd5 --- /dev/null +++ b/examples/gpmdk/run/water/latte.in @@ -0,0 +1,74 @@ +LATTE INPUT FILE +================ +#This input file resumes the content of MDcontroller and TBparam/control.in +#The parser will only read it if it is present inside the running folder. +#In case this file is not present Latte will read the two files as original. +#The order of the kewords is not important in this file. + +#General controls +CONTROL{ + XCONTROL= 1 + BASISTYPE= NONORTHO + PARAMPATH= "./latteTBparams" + DEBUGON= 0 + FERMIM= 6 + CGORLIB= 1 CGTOL= 1.0e-6 + KBT= 0.0 + NORECS= 1 + ENTROPYKIND= 1 + PPOTON= 1 VDWON= 0 + SPINON= 0 SPINTOL= 1.0e-4 + ELECTRO= 0 ELECMETH= 1 ELEC_ETOL= 0.001 ELEC_QTOL= 1.0e-4 + COULACC= 1.0e-6 COULCUT= -500.0 COULR1= 500.0 + MAXSCF= 250 + BREAKTOL= 1.0E-12 MINSP2ITER= 22 SP2CONV= REL + FULLQCONV= 1 QITER= 0 + QMIX= 0.25 SPINMIX= 0.25 MDMIX= 0.25 + ORDERNMOL= 0 + SPARSEON= 0 THRESHOLDON= 1 NUMTHRESH= 1.0e-6 FILLINSTOP= 100 BLKSZ= 4 + MSPARSE= 3000 + LCNON= 0 LCNITER= 4 CHTOL= 0.01 + SKIN= 1.0 + RELAX= 0 RELAXTYPE= SD MAXITER= 100 RLXFTOL= 0.00001 + MDON= 0 + PBCON= 1 + RESTART= 0 + CHARGE= 0 + XBO= 1 + XBODISON= 1 + XBODISORDER= 5 + NGPU= 2 + KON= 0 + COMPFORCE= 1 + DOSFIT= 0 INTS2FIT= 1 BETA= 1000.0 NFITSTEP= 5000 QFIT= 0 MCSIGMA= 0.2 + PPFITON= 0 + ALLFITON= 0 + PPSTEP= 500 BISTEP= 500 PP2FIT= 2 BINT2FIT= 6 + PPBETA= 1000.0 PPSIGMA= 0.01 PPNMOL= 10 PPNGEOM= 200 + PARREP= 0 + ER= 1.0 +} + +#Controls for QMD +MDCONTROL{ + MAXITER= 20 + UDNEIGH= 1 + DT= 0.25 + TEMPERATURE= 1.0e-30 RNDIST= GAUSSIAN SEEDINIT= UNIFORM + DUMPFREQ= 250 + RSFREQ= 500 + WRTFREQ= 1 + TOINITTEMP5= 1 + THERMPER= 500 + THERMRUN= 50000 + NVTON= 0 NPTON= 0 AVEPER= 1000 FRICTION= 1000.0 SEED= 54 + PTARGET= 0.0 NPTTYPE= ISO + SHOCKON= 0 + SHOCKSTART= 100000 + SHOCKDIR= 1 + UPARTICLE= 500.0 USHOCK= -4590.0 C0= 1300.0 + MDADAPT= 0 + GETHUG= 0 E0= -795.725 V0= 896.984864 P0= 0.083149 +} + + diff --git a/examples/gpmdk/run/water/run.sh b/examples/gpmdk/run/water/run.sh new file mode 100755 index 00000000..0696145d --- /dev/null +++ b/examples/gpmdk/run/water/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + +#RUN="$PWD/../build_hackathon/gpmdk" +#OMP_NUM_THREADS=32 mpiexec -np 3 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpirun -np 1 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpiexec -np 1 $RUN input.in | tee out + +#OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out +OMP_NUM_THREADS=32 srun -n 4 -c 32 bash wrapper.sh +#OMP_NUM_THREADS=64 $RUN input.in | tee out +#OMP_NUM_THREADS=36 $RUN input.in | tee out +#$RUN input.in | tee out + diff --git a/examples/gpmdk/run/water/run_single.sh b/examples/gpmdk/run/water/run_single.sh new file mode 100755 index 00000000..5b5bca02 --- /dev/null +++ b/examples/gpmdk/run/water/run_single.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + +RUN="$PWD/../../build_hackathon/gpmdk" +#OMP_NUM_THREADS=32 mpiexec -np 3 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpirun -np 1 $RUN input.in | tee out +#OMP_NUM_THREADS=36 mpiexec -np 1 $RUN input.in | tee out + +#OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out +#OMP_NUM_THREADS=64 $RUN input.in >& out +OMP_NUM_THREADS=72 srun -n 1 -c 72 $RUN input.in >& out +#OMP_NUM_THREADS=36 $RUN input.in | tee out +#$RUN input.in | tee out + diff --git a/examples/gpmdk/run/water/sbatch.sh b/examples/gpmdk/run/water/sbatch.sh new file mode 100755 index 00000000..166b6caf --- /dev/null +++ b/examples/gpmdk/run/water/sbatch.sh @@ -0,0 +1,21 @@ +#!/bin/bash +##SBATCH -p gpu +##SBATCH -p gpu_debug +##SBATCH --reservation=gpu_debug +##SBATCH --qos=debug +##SBATCH --time 16:00:00 +##SBATCH --time 2:00:00 +#SBATCH -N 4 +##SBATCH -A ichelp_g +##SBATCH -A w23_macroqmd_g + +cd ${PWD} + +source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh + +#export MPICH_ALLREDUCE_NO_SMP=1 +export MPICH_SMP_SINGLE_COPY_MODE=NONE + +#OMP_NUM_THREADS=32 srun -n 8 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh +OMP_NUM_THREADS=72 srun -n 16 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh + diff --git a/examples/gpmdk/run/water/wrapper.sh b/examples/gpmdk/run/water/wrapper.sh new file mode 100644 index 00000000..3cc5b863 --- /dev/null +++ b/examples/gpmdk/run/water/wrapper.sh @@ -0,0 +1,10 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) + +RUN="$PWD/../../build_hackathon/gpmdk" +#stdbuf -o0 $RUN input_ICH.in |& tee out_ICH +if [ $SLURM_PROCID -eq 0 ]; then + stdbuf -o0 $RUN input.in >& out +else + $RUN input.in >& /dev/null +fi diff --git a/scripts/setenv_venado.sh b/scripts/setenv_venado.sh new file mode 100644 index 00000000..09f1c773 --- /dev/null +++ b/scripts/setenv_venado.sh @@ -0,0 +1,3 @@ +module switch PrgEnv-cray PrgEnv-gnu +export PKG_CONFIG_PATH=/opt/cray/xpmem/2.8.2-1.0_5.1__g84a27a5.shasta/lib64/pkgconfig:$PKG_CONFIG_PATH +module load cudatoolkit From 7525e35f634253458c58fddaf9c74dd8d2addf97 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 5 Jul 2024 11:51:29 -0600 Subject: [PATCH 08/69] Bug fix --- examples/gpmdk/src/gpmdcov_part.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index 2209d241..1ed1e572 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -42,7 +42,6 @@ subroutine gpmdcov_Part(ipreMD) if(gpat%TotalParts > 1)then call gpmdcov_msI("gpmdcov_Part","In prg_get_covgraph_h ...",lt%verbose,myRank) mls_ii = mls() -#ifdef DO_MPI call prg_get_covgraph_h(sy,nl%nnStruct,nl%nrnnstruct,gsp2%nlgcut,graph_h,myMdim,lt%verbose) call gpmdcov_msII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) From 17ad747945dcf18e9d34241cb4359b575629badf Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 5 Jul 2024 11:54:01 -0600 Subject: [PATCH 09/69] Add TrpCage example for gpmdk --- examples/gpmdk/run/TrpCage/input.dat | 1 + examples/gpmdk/run/TrpCage/input.in | 208 + .../gpmdk/run/TrpCage/minimized_TrpCage.dat | 8018 +++++++++++++++++ examples/gpmdk/run/TrpCage/sbatch.sh | 19 + examples/gpmdk/run/TrpCage/sbatch_profile.sh | 18 + examples/gpmdk/run/TrpCage/wrapper.sh | 12 + examples/gpmdk/run/TrpCage/wrapper_profile.sh | 12 + 7 files changed, 8288 insertions(+) create mode 120000 examples/gpmdk/run/TrpCage/input.dat create mode 100644 examples/gpmdk/run/TrpCage/input.in create mode 100644 examples/gpmdk/run/TrpCage/minimized_TrpCage.dat create mode 100755 examples/gpmdk/run/TrpCage/sbatch.sh create mode 100755 examples/gpmdk/run/TrpCage/sbatch_profile.sh create mode 100644 examples/gpmdk/run/TrpCage/wrapper.sh create mode 100644 examples/gpmdk/run/TrpCage/wrapper_profile.sh diff --git a/examples/gpmdk/run/TrpCage/input.dat b/examples/gpmdk/run/TrpCage/input.dat new file mode 120000 index 00000000..123bac58 --- /dev/null +++ b/examples/gpmdk/run/TrpCage/input.dat @@ -0,0 +1 @@ +minimized_TrpCage.dat \ No newline at end of file diff --git a/examples/gpmdk/run/TrpCage/input.in b/examples/gpmdk/run/TrpCage/input.in new file mode 100644 index 00000000..165ce61f --- /dev/null +++ b/examples/gpmdk/run/TrpCage/input.in @@ -0,0 +1,208 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + #StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.25 + #TimeStep= 0.00 + MDSteps= 1000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + #CoordsFile= coords_300.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb + #CoordsFile= minimized_TrpCage.dat + #CoordsFile= Mac1_latte.dat + #CoordsFile= ICH.dat + CoordsFile= input.dat + NlistEach= 1 + #BMLDistributionType= Distributed + MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn + EFermi= -0.0 + #kBT= 0.04308695 + kBT= 0.025 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + #PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + PartitionType= METIS + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 128 + #PartitionCount= 8 + #PartitionCount= 512 + #PartitionCount= 16 + #PartitionCount= 2 + #PartitionCount= 8 + #PartitionCount= 1024 + #PartitionCount= 800 + #PartitionCount= 1600 + #PartitionCount= 32 + #PartitionCount= 64 + PartitionCount= 256 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.0005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= 4000 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks + BuildAlways= F + RankNUpdate= 3 + KernelMixing= T + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 2 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + TrajectoryFormat= XYZ + WriteCoordsEach= 1 + InitialTemperature= 300 + DumpEach= 100 + RestartFromDump= F + #MinimizationSteps= 500 + LangevinMethod= Siva + LangevinDynamics= F + LangevinGamma= 0.01 + #SMDForceConstantStart= 0.1 + #SMDForceConstantEnd= 0.1 + #SMDR0= 2.0 + #SMDNumPairs= 1 + #SMDPairs[ + #127 317 + #] + #SMDAtomIndex1= 324 + #SMDAtomIndex2= 315 + UseSMD= F + HtoD= T + UseVectSKBlock= F + ReplicateX= 2 + ReplicateY= 2 + ReplicateZ= 1 +} diff --git a/examples/gpmdk/run/TrpCage/minimized_TrpCage.dat b/examples/gpmdk/run/TrpCage/minimized_TrpCage.dat new file mode 100644 index 00000000..ee38c78f --- /dev/null +++ b/examples/gpmdk/run/TrpCage/minimized_TrpCage.dat @@ -0,0 +1,8018 @@ + 8014 + 43.92300 0.00000 0.00000 + 0.00000 43.92300 0.00000 + 0.00000 0.00000 43.92300 + N 10.55400 -10.96800 -16.17800 + H 10.27800 -11.87200 -15.74600 + H 9.82400 -10.24800 -16.00400 + H 11.42500 -10.63700 -15.71700 + C 10.70000 -11.20100 -17.65500 + H 11.25100 -12.12100 -17.77700 + C 9.29500 -11.33300 -18.31800 + H 8.66700 -10.44200 -18.10100 + H 9.40000 -11.42700 -19.42000 + C 8.57000 -12.57900 -17.81800 + O 9.08200 -13.29600 -16.95700 + N 7.35900 -12.84500 -18.34500 + H 6.92200 -12.18500 -18.95700 + H 6.85600 -13.64200 -18.01000 + C 11.51600 -10.14500 -18.39200 + O 12.26100 -10.47100 -19.31200 + N 11.43100 -8.85300 -18.00200 + H 10.82200 -8.57200 -17.26500 + C 12.05800 -7.73700 -18.70200 + H 11.63100 -7.71200 -19.69400 + C 11.68900 -6.39300 -18.00600 + H 12.17300 -5.53300 -18.51500 + H 10.59700 -6.27100 -18.17100 + C 11.91900 -6.28800 -16.47400 + H 11.95400 -7.30300 -16.02300 + C 13.22600 -5.56700 -16.11200 + H 14.11300 -6.05900 -16.56500 + H 13.18900 -4.51300 -16.46200 + H 13.36500 -5.55200 -15.01000 + C 10.75200 -5.54600 -15.80000 + H 10.67700 -4.50500 -16.18100 + H 9.78600 -6.05700 -15.99800 + H 10.90400 -5.50500 -14.70000 + C 13.56800 -7.83900 -18.92300 + O 14.06500 -7.54900 -20.01200 + N 14.33200 -8.29000 -17.90700 + H 13.91200 -8.45900 -17.01800 + C 15.76400 -8.52400 -17.99800 + H 16.21000 -7.61800 -18.37900 + C 16.31100 -8.81900 -16.57900 + H 15.77800 -8.18500 -15.83900 + H 16.16100 -9.87700 -16.27400 + C 17.77600 -8.48000 -16.48500 + C 18.76700 -9.43600 -16.77300 + H 18.47800 -10.41800 -17.12000 + C 20.12200 -9.14100 -16.56700 + H 20.87300 -9.89900 -16.73800 + C 20.49100 -7.89500 -16.05300 + O 21.81500 -7.65800 -15.64500 + H 21.73800 -7.41600 -14.71900 + C 18.16800 -7.21600 -16.01100 + H 17.42100 -6.47500 -15.76700 + C 19.51900 -6.92500 -15.79000 + H 19.80600 -5.97200 -15.37200 + C 16.12500 -9.64800 -18.97600 + O 17.07400 -9.55000 -19.74900 + N 15.34200 -10.74700 -18.99100 + H 14.56800 -10.80900 -18.36500 + C 15.50000 -11.87600 -19.90500 + H 16.51200 -12.23900 -19.80400 + C 14.53100 -13.01700 -19.57100 + H 13.49200 -12.69200 -19.79300 + C 14.82900 -14.24900 -20.46000 + H 14.64100 -14.03400 -21.53400 + H 15.88800 -14.56800 -20.35200 + H 14.17800 -15.10000 -20.17100 + C 14.56500 -13.39600 -18.06600 + H 13.88000 -14.25900 -17.91900 + H 14.15600 -12.56400 -17.45400 + C 15.95000 -13.77500 -17.52400 + H 15.87800 -14.06600 -16.45400 + H 16.38000 -14.63200 -18.08400 + H 16.65600 -12.91900 -17.59000 + C 15.33800 -11.45300 -21.36000 + O 16.10000 -11.85400 -22.23900 + N 14.35000 -10.57600 -21.63200 + H 13.72700 -10.30600 -20.90200 + C 14.12200 -9.97700 -22.93600 + H 13.99400 -10.78600 -23.64000 + C 12.84100 -9.10100 -22.88400 + H 12.15300 -9.53200 -22.12600 + H 13.08600 -8.07500 -22.53800 + C 12.07200 -9.02300 -24.22700 + H 11.30000 -8.22600 -24.16700 + H 12.77200 -8.74800 -25.04500 + C 11.40800 -10.35600 -24.59400 + O 12.04600 -11.30200 -25.06400 + N 10.07400 -10.44200 -24.40000 + H 9.58500 -9.65300 -24.02800 + H 9.59500 -11.24800 -24.74700 + C 15.32500 -9.14300 -23.40200 + O 15.75100 -9.21000 -24.55100 + N 15.94700 -8.36400 -22.49300 + H 15.57500 -8.28100 -21.57200 + C 17.20600 -7.68400 -22.75700 + H 17.06900 -7.08800 -23.64700 + C 17.52100 -6.72100 -21.57500 + H 16.69300 -5.98200 -21.52000 + H 17.49900 -7.29300 -20.62300 + C 18.82200 -5.98300 -21.64300 + C 19.04300 -4.86400 -22.39300 + H 18.27600 -4.38700 -22.98500 + N 20.35000 -4.45400 -22.23800 + H 20.77800 -3.66800 -22.62600 + C 20.98200 -5.27800 -21.35900 + C 20.05100 -6.26300 -20.95200 + C 20.43800 -7.23300 -20.02800 + H 19.74800 -7.99400 -19.69400 + C 21.73500 -7.18300 -19.51300 + H 22.04200 -7.91300 -18.77900 + C 22.27000 -5.23400 -20.85500 + H 22.96400 -4.46400 -21.15900 + C 22.63300 -6.19800 -19.91100 + H 23.62000 -6.17800 -19.47400 + C 18.37700 -8.63000 -23.06900 + O 19.15300 -8.42000 -24.00400 + N 18.51500 -9.73000 -22.30000 + H 17.91500 -9.85700 -21.51300 + C 19.49800 -10.76900 -22.55800 + H 20.46000 -10.28100 -22.61400 + C 19.55300 -11.82200 -21.41600 + H 18.53500 -12.24100 -21.26900 + H 20.22300 -12.65700 -21.71400 + C 20.07900 -11.29400 -20.06000 + H 19.43100 -10.45500 -19.73000 + C 20.00500 -12.39300 -18.98800 + H 18.96300 -12.76300 -18.87700 + H 20.66600 -13.24400 -19.25900 + H 20.34100 -11.99800 -18.00600 + C 21.52000 -10.76500 -20.16200 + H 22.19300 -11.55200 -20.56500 + H 21.57300 -9.87800 -20.82900 + H 21.89200 -10.46200 -19.16000 + C 19.34700 -11.47600 -23.90300 + O 20.34200 -11.67700 -24.60000 + N 18.11600 -11.84700 -24.34000 + H 17.31700 -11.69900 -23.76200 + C 17.89800 -12.53500 -25.61600 + H 18.55500 -13.39200 -25.60300 + C 16.47300 -13.15300 -25.80700 + H 16.50600 -13.86600 -26.65700 + H 16.28800 -13.75700 -24.89300 + C 15.25700 -12.22000 -25.99500 + H 15.34100 -11.38300 -25.26900 + H 14.34500 -12.79200 -25.72000 + C 15.06600 -11.67300 -27.42700 + H 14.98000 -12.52300 -28.13800 + H 15.97400 -11.11100 -27.73500 + C 13.90900 -10.67200 -27.61600 + H 14.02900 -10.11600 -28.57100 + H 13.87900 -9.93900 -26.78200 + N 12.61800 -11.37600 -27.69400 + H 12.68200 -12.11700 -28.42100 + H 12.40100 -11.76400 -26.75400 + H 11.84100 -10.73000 -27.94000 + C 18.34600 -11.72000 -26.82500 + O 18.86900 -12.26100 -27.79800 + N 18.19100 -10.37900 -26.76900 + H 17.71100 -9.96500 -25.99900 + C 18.59900 -9.46400 -27.82000 + H 18.33800 -9.89600 -28.77500 + C 17.87000 -8.10400 -27.63600 + H 17.90800 -7.77900 -26.57400 + H 18.32700 -7.31400 -28.26800 + C 16.42100 -8.18200 -28.08100 + O 16.11500 -8.93400 -29.04100 + O 15.60200 -7.38900 -27.54900 + C 20.11400 -9.25200 -27.86400 + O 20.65100 -8.74100 -28.84900 + N 20.84200 -9.64800 -26.79700 + H 20.35800 -10.07600 -26.03800 + C 22.30200 -9.71300 -26.78200 + H 22.71300 -9.26500 -27.67400 + H 22.56500 -10.75600 -26.68800 + C 22.92900 -9.00100 -25.61800 + O 24.15600 -8.92900 -25.52500 + N 22.11700 -8.46300 -24.68100 + H 21.12700 -8.53500 -24.77400 + C 22.59600 -7.70300 -23.52800 + H 23.14700 -8.40100 -22.91500 + H 21.71900 -7.32000 -23.02900 + C 23.45400 -6.49400 -23.88200 + O 23.01900 -5.69000 -24.71000 + N 24.66400 -6.27600 -23.34200 + C 25.23900 -7.04900 -22.23500 + H 24.47100 -7.44100 -21.53500 + H 25.82800 -7.89100 -22.66000 + C 25.44600 -5.06500 -23.60900 + H 24.81800 -4.21400 -23.38800 + C 26.66000 -5.18200 -22.66800 + H 27.01600 -4.18900 -22.31900 + H 27.50300 -5.71200 -23.16000 + C 26.15600 -6.05700 -21.52100 + H 25.56300 -5.43900 -20.81300 + H 26.98100 -6.55800 -20.97000 + C 25.88400 -4.94400 -25.05700 + O 26.17400 -3.84200 -25.51900 + N 25.92600 -6.06700 -25.79500 + H 25.69300 -6.93700 -25.36900 + C 26.37400 -6.13700 -27.17900 + H 27.10300 -5.36200 -27.36400 + C 27.03300 -7.50800 -27.48400 + H 26.29200 -8.32400 -27.34200 + H 27.39900 -7.54300 -28.53200 + O 28.13700 -7.71800 -26.60100 + H 28.39800 -8.63800 -26.68700 + C 25.21900 -5.93200 -28.14600 + O 25.33800 -6.18300 -29.34200 + N 24.06400 -5.45400 -27.64100 + H 23.99000 -5.32200 -26.65500 + C 22.85900 -5.21500 -28.42500 + H 22.96300 -5.63700 -29.41300 + C 21.65300 -5.86500 -27.69900 + H 21.96000 -6.88500 -27.38000 + H 21.39400 -5.30800 -26.77400 + O 20.50900 -5.98600 -28.54300 + H 20.55500 -6.87100 -28.91200 + C 22.57100 -3.72300 -28.57500 + O 21.53800 -3.31600 -29.10800 + N 23.49200 -2.84900 -28.11400 + H 24.37800 -3.19100 -27.81200 + C 23.36500 -1.39300 -28.18700 + H 23.20100 -1.12300 -29.22000 + H 24.28200 -0.97600 -27.79700 + C 22.25200 -0.77000 -27.38400 + O 21.65800 0.22200 -27.80700 + N 21.93200 -1.32200 -26.20000 + H 22.41300 -2.12700 -25.86100 + C 20.85500 -0.81600 -25.37000 + H 20.75500 0.24200 -25.56300 + C 19.52700 -1.53200 -25.74400 + H 18.67700 -1.04700 -25.21800 + H 19.37500 -1.36800 -26.83200 + C 19.52000 -3.04800 -25.44500 + H 19.40000 -3.14100 -24.34400 + H 20.50400 -3.49600 -25.70000 + C 18.41600 -3.88000 -26.11400 + H 17.40700 -3.44700 -25.94300 + H 18.41800 -4.91300 -25.70600 + N 18.74900 -3.96400 -27.56800 + H 19.43000 -4.62600 -27.88300 + C 18.17500 -3.27400 -28.53800 + N 17.15200 -2.47100 -28.37200 + H 16.74300 -2.35300 -27.46800 + H 16.70600 -2.10900 -29.19100 + N 18.70900 -3.37200 -29.73900 + H 19.65500 -3.69700 -29.75400 + H 18.38500 -2.76100 -30.46000 + C 21.20900 -1.02300 -23.88900 + O 21.73700 -2.08500 -23.54100 + N 20.99900 -0.05600 -22.98800 + C 20.23300 1.16900 -23.26200 + H 20.88500 1.88400 -23.80900 + H 19.31500 0.96200 -23.85400 + C 21.29600 -0.19200 -21.55900 + H 22.33800 -0.46500 -21.47900 + C 21.00200 1.21200 -21.00000 + H 21.89800 1.85200 -21.14600 + H 20.73300 1.21700 -19.92200 + C 19.87000 1.73200 -21.88700 + H 19.79300 2.84000 -21.88500 + H 18.90100 1.30300 -21.55200 + C 20.48300 -1.27900 -20.84800 + O 19.37300 -1.55700 -21.31400 + N 20.94600 -1.92500 -19.77000 + C 22.22200 -1.62100 -19.10600 + H 23.03200 -2.16700 -19.63500 + H 22.44400 -0.53200 -19.08500 + C 20.12300 -2.82600 -18.96100 + H 19.82000 -3.63000 -19.61400 + C 21.07300 -3.30700 -17.84900 + H 21.62000 -4.20500 -18.20800 + H 20.55300 -3.56900 -16.90300 + C 22.06100 -2.15100 -17.67800 + H 23.02500 -2.47100 -17.22800 + H 21.60000 -1.36400 -17.04400 + C 18.85700 -2.18000 -18.39100 + O 18.91500 -0.99400 -18.05300 + N 17.72500 -2.86600 -18.25300 + C 17.45000 -4.17100 -18.86400 + H 17.50500 -4.06100 -19.96800 + H 18.17400 -4.93800 -18.51500 + C 16.54700 -2.29600 -17.61900 + H 16.44800 -1.24900 -17.86100 + C 15.40000 -3.13400 -18.20800 + H 15.11700 -2.69800 -19.19000 + H 14.49900 -3.15400 -17.55900 + C 16.02300 -4.51400 -18.43900 + H 15.48000 -5.11200 -19.20200 + H 16.05300 -5.07400 -17.48100 + C 16.65300 -2.42500 -16.10300 + O 17.15600 -3.42600 -15.58900 + N 16.21300 -1.38800 -15.37600 + H 15.78700 -0.60800 -15.82600 + C 16.27900 -1.30400 -13.92800 + H 17.00100 -2.01300 -13.54900 + C 16.66800 0.11700 -13.44100 + H 15.85100 0.82900 -13.68600 + H 16.80300 0.10800 -12.33800 + O 17.86600 0.58800 -14.06800 + H 18.52800 -0.10300 -13.98300 + C 14.90700 -1.63400 -13.30000 + O 13.88800 -1.02300 -13.72200 + O 14.84300 -2.47400 -12.36500 + C -5.40000 -7.01600 -3.68900 + O -4.66900 -7.10000 -4.73400 + O -4.95200 -7.66400 -2.52600 + O -6.48800 -6.34300 -3.68200 + H -4.06700 -7.99900 -2.68900 + C 3.52600 -15.70400 10.25200 + O 2.56900 -16.48500 9.90700 + O 3.45600 -15.05200 11.49300 + O 4.55700 -15.54800 9.51700 + H 2.70300 -15.39000 11.98200 + C -3.01800 18.91300 2.27100 + O -4.05500 18.19200 2.17500 + O -2.94800 19.73700 3.38900 + O -2.07800 18.89800 1.40300 + H -3.66300 19.46900 3.97100 + C -3.05600 15.74000 -17.98300 + O -3.54300 14.58100 -18.24500 + O -2.02800 15.81700 -17.04200 + O -3.53600 16.79800 -18.50800 + H -1.83400 14.92700 -16.73700 + C -17.63000 17.04500 16.56800 + O -18.14500 15.88400 16.62500 + O -16.70500 17.39400 17.53800 + O -17.97200 17.89200 15.66300 + H -16.52000 16.62900 18.08600 + H 6.84500 6.79200 6.84900 + N 6.96700 7.82100 6.75700 + H 6.11900 8.29200 7.13100 + H 7.78600 8.08900 7.34000 + H 7.11400 8.04700 5.75300 + H 1.59000 -15.32900 -12.06800 + N 2.22600 -14.66400 -11.58400 + H 2.86700 -14.22200 -12.27400 + H 1.64800 -13.94300 -11.10600 + H 2.81000 -15.16800 -10.88700 + H -5.90000 -5.75200 -19.16700 + N -5.27000 -5.18100 -18.56900 + H -5.21100 -5.55500 -17.60100 + H -4.32600 -5.17600 -19.00500 + H -5.65800 -4.21800 -18.51200 + H -15.42600 -22.17700 -6.87400 + N -14.62200 -21.51800 -6.87600 + H -14.36700 -21.24900 -5.90500 + H -13.80300 -21.96400 -7.33700 + H -14.88800 -20.66800 -7.41300 + O -10.52200 -12.67400 -16.14000 + H -10.62000 -11.72300 -16.19400 + H -10.96200 -13.00600 -16.92200 + O -10.73500 -11.94900 -12.12900 + H -10.72200 -12.50800 -11.35200 + H -11.23500 -12.44800 -12.77500 + O -18.77100 -13.83900 -11.03800 + H -18.66500 -14.34600 -10.23400 + H -17.93400 -13.38900 -11.15100 + O -6.10800 -9.91900 -13.68300 + H -6.83700 -10.51400 -13.50600 + H -6.14100 -9.28100 -12.97000 + O -16.93300 -8.06300 -13.29700 + H -17.42200 -8.88400 -13.25000 + H -16.10600 -8.25100 -12.85400 + O -20.41000 -13.53000 -17.13600 + H -20.32800 -12.94200 -16.38500 + H -21.13000 -14.11700 -16.90400 + O -19.99600 -12.15200 -14.75800 + H -19.71900 -12.97100 -14.34700 + H -19.42700 -11.48600 -14.37200 + O -9.77600 -6.40000 -13.54400 + H -10.47800 -6.89000 -13.97300 + H -9.63100 -6.86100 -12.71800 + O -12.31100 -9.69900 -11.76300 + H -11.81700 -10.44400 -12.10700 + H -11.85500 -9.46100 -10.95600 + O -3.91700 -17.83800 -10.55200 + H -4.33400 -17.50600 -9.75700 + H -3.93800 -17.09900 -11.16000 + O -10.80500 -15.71800 -12.52200 + H -10.77800 -16.58700 -12.12200 + H -10.40100 -15.83600 -13.38200 + O -15.53000 -17.66900 -13.77300 + H -15.60600 -16.83500 -14.23700 + H -16.07800 -18.27300 -14.27500 + O -7.78100 -17.80700 -15.53000 + H -7.61600 -17.84600 -14.58800 + H -7.00100 -17.38800 -15.89300 + O -19.42700 -14.57900 -13.68000 + H -19.10100 -14.53600 -12.78100 + H -19.04700 -15.38300 -14.03500 + O -14.80800 -13.40900 -13.70900 + H -13.85300 -13.44900 -13.76100 + H -15.10600 -14.19000 -14.17400 + O -5.09900 -16.67200 -8.41200 + H -5.49700 -15.80800 -8.51500 + H -4.40300 -16.54100 -7.76800 + O -17.65500 -19.22200 -9.90800 + H -17.59700 -18.44500 -10.46500 + H -18.54900 -19.20400 -9.56500 + O -10.90700 -10.02400 -16.81700 + H -11.30300 -9.91200 -17.68100 + H -10.92800 -9.15000 -16.42700 + O -14.87800 -10.32800 -10.81300 + H -15.26500 -9.53700 -10.43600 + H -14.07900 -10.02300 -11.24300 + O -7.84800 -17.57500 -12.81900 + H -8.67300 -17.85000 -12.41900 + H -7.44900 -16.99500 -12.17200 + O -17.26300 -17.12000 -11.66600 + H -16.85800 -16.41800 -11.15800 + H -16.59700 -17.36600 -12.30800 + O -16.10900 -8.27200 -9.41500 + H -16.30000 -7.34400 -9.55000 + H -16.89800 -8.62200 -9.00200 + O -19.77400 -8.75500 -17.18800 + H -20.62800 -8.76600 -16.75600 + H -19.34700 -7.96500 -16.85500 + O -6.71500 -8.19800 -11.59200 + H -7.66000 -8.12100 -11.46000 + H -6.39600 -7.29600 -11.57400 + O -6.19000 -19.70400 -9.94800 + H -7.09300 -19.39900 -9.86900 + H -5.74200 -19.00800 -10.42900 + O -14.66100 -21.66300 -12.65500 + H -14.57500 -22.61600 -12.61100 + H -15.54700 -21.48800 -12.33700 + O -8.39000 -6.17000 -15.90800 + H -7.72100 -6.85400 -15.90700 + H -8.69100 -6.12700 -15.00100 + O -6.55800 -8.20800 -15.79200 + H -6.75900 -8.82300 -16.49800 + H -6.35600 -8.76400 -15.03900 + O -5.88400 -11.63500 -8.41400 + H -5.89700 -11.61800 -9.37100 + H -5.83800 -10.71300 -8.15800 + O -11.76600 -13.47800 -18.48200 + H -12.49400 -12.85700 -18.47500 + H -11.57000 -13.60800 -19.41000 + O -10.90300 -13.87900 -21.08300 + H -10.07200 -14.03200 -20.63400 + H -10.83200 -14.37700 -21.89700 + O -3.62600 -13.10300 -10.12400 + H -4.42900 -12.65600 -10.39100 + H -2.92700 -12.63400 -10.58000 + O -7.91800 -12.81200 -17.21400 + H -8.73400 -12.78700 -16.71500 + H -7.29500 -13.24900 -16.63300 + O -9.10900 -14.28600 -7.97600 + H -9.05200 -13.51800 -7.40800 + H -9.63700 -13.99500 -8.72000 + O -16.28200 -11.76700 -15.38300 + H -16.68400 -12.32200 -16.05100 + H -15.72300 -12.36200 -14.88200 + O -14.11400 -7.95600 -12.85900 + H -14.11400 -7.98700 -13.81600 + H -13.39700 -8.53600 -12.60100 + O -16.30500 -12.57800 -11.55100 + H -15.76300 -12.88400 -12.27800 + H -15.82500 -11.83200 -11.19100 + O -9.41400 -7.66600 -11.10800 + H -9.52100 -6.83200 -10.65000 + H -9.94700 -8.28400 -10.60900 + O -14.91200 -8.11900 -6.88500 + H -15.64400 -7.62600 -6.51400 + H -15.18900 -8.32400 -7.77800 + O -8.52700 -14.04700 -19.64500 + H -8.31400 -13.66400 -18.79400 + H -8.14700 -13.44200 -20.28200 + O -17.73200 -12.32200 -6.65600 + H -17.76700 -11.53800 -6.10800 + H -18.52400 -12.80900 -6.42800 + O -10.92000 -7.31800 -16.17000 + H -10.01700 -7.00600 -16.21100 + H -11.43400 -6.62300 -16.58200 + O -17.01500 -6.43300 -6.21800 + H -16.85200 -5.91900 -7.00800 + H -17.96000 -6.36700 -6.07900 + O -3.82700 -21.73200 -18.10400 + H -3.96700 -20.90900 -17.63500 + H -3.15400 -21.52700 -18.75300 + O -17.65000 -12.87300 -17.54200 + H -17.92000 -12.04600 -17.94100 + H -18.47000 -13.32500 -17.34600 + O -3.09300 -5.81100 -20.23200 + H -3.47300 -5.70500 -21.10400 + H -2.42600 -6.48800 -20.34100 + O -20.46100 -10.73800 -8.05600 + H -21.08100 -10.06500 -8.33800 + H -20.62100 -11.47400 -8.64600 + O -20.64800 -7.47300 -11.02200 + H -21.14300 -8.05600 -10.44600 + H -19.96700 -7.10300 -10.46000 + O -12.48800 -6.87400 -6.86000 + H -12.57200 -6.12800 -6.26700 + H -13.33400 -7.31900 -6.80900 + O -11.85400 -4.97600 -17.39300 + H -12.07500 -4.89500 -18.32100 + H -12.58300 -4.55500 -16.93600 + O -17.78000 -4.16900 -15.72200 + H -17.42800 -4.45900 -14.88100 + H -18.42000 -4.84000 -15.95900 + O -9.14800 -4.89900 -20.25100 + H -8.34400 -5.40800 -20.35400 + H -9.09100 -4.53800 -19.36600 + O -9.21500 -4.12100 -17.52300 + H -10.15100 -4.31800 -17.49000 + H -8.81300 -4.77300 -16.94800 + O -6.59600 -15.38600 -5.00300 + H -7.45700 -14.99900 -4.84500 + H -6.77600 -16.16100 -5.53600 + O -18.27500 -9.00100 -7.83100 + H -17.92900 -9.20000 -6.96100 + H -18.95300 -9.66100 -7.97700 + O -17.29200 -21.11100 -11.88700 + H -18.09000 -20.81800 -12.32800 + H -17.23500 -20.56400 -11.10400 + O -10.33800 -21.31600 -9.30000 + H -11.19300 -21.59400 -8.97300 + H -10.34800 -21.55100 -10.22800 + O -9.54100 -17.00800 -7.62800 + H -9.92400 -17.15300 -6.76300 + H -9.34000 -16.07200 -7.64800 + O -7.83500 -11.96400 -21.26600 + H -8.34600 -11.31600 -20.78000 + H -6.93700 -11.63300 -21.23600 + O -12.55800 -18.89200 -20.54400 + H -13.32800 -18.51500 -20.97100 + H -12.89700 -19.28100 -19.73800 + O -16.08500 -15.13800 -10.14400 + H -16.72200 -15.03800 -9.43700 + H -15.93000 -14.24600 -10.45400 + O -16.01300 -15.35700 -15.21800 + H -16.94800 -15.55900 -15.21700 + H -15.69300 -15.70700 -16.05000 + O -10.57500 -15.83900 -17.53900 + H -10.94400 -15.04500 -17.92700 + H -9.92600 -16.13700 -18.17600 + O -11.45900 -14.55400 -5.72200 + H -11.87800 -15.20600 -5.16000 + H -10.58100 -14.45000 -5.35400 + O -9.63100 -15.85600 -14.96000 + H -10.02700 -15.67400 -15.81200 + H -9.03700 -16.58900 -15.12300 + O -18.37600 -15.09700 -8.56700 + H -18.74000 -14.52900 -7.88800 + H -18.52800 -15.98500 -8.24200 + O -15.30300 -18.46100 -21.22500 + H -15.64400 -18.76900 -20.38500 + H -15.69800 -19.04400 -21.87300 + O -21.29600 -17.36600 -15.09900 + H -20.66300 -17.36300 -14.38100 + H -20.76200 -17.45300 -15.88900 + O -20.06700 -17.12400 -17.71700 + H -19.42000 -16.74700 -18.31400 + H -20.90500 -17.00300 -18.16400 + O -10.82000 -14.05400 -10.27500 + H -10.77400 -14.63100 -11.03700 + H -11.42300 -14.49500 -9.67600 + O -12.86800 -18.35200 -14.16300 + H -13.76100 -18.07600 -13.95600 + H -12.57700 -18.82500 -13.38400 + O -18.39300 -16.71800 -15.42300 + H -18.87200 -16.72900 -16.25100 + H -17.99700 -17.58700 -15.36300 + O -15.34400 -16.75100 -17.45000 + H -14.41000 -16.91900 -17.32800 + H -15.41300 -16.42000 -18.34600 + O -10.31700 -18.27300 -11.64300 + H -9.83800 -18.56700 -10.86800 + H -11.01200 -18.92100 -11.75600 + O -15.90600 -15.96900 -20.01600 + H -16.86100 -16.03400 -20.01700 + H -15.61900 -16.62100 -20.65600 + O -5.84500 -3.76500 -4.08600 + H -4.99700 -3.63900 -3.66000 + H -6.00500 -4.70700 -4.02700 + O -17.37400 -9.84300 -5.31900 + H -16.44000 -10.02900 -5.21600 + H -17.63400 -9.45400 -4.48400 + O -12.04300 -13.60600 -13.93600 + H -11.47700 -13.41200 -14.68300 + H -11.66300 -14.39300 -13.54500 + O -17.19600 -19.27500 -15.32300 + H -17.99400 -19.80100 -15.36800 + H -16.48800 -19.89900 -15.48700 + O -14.43200 -16.83600 -8.70800 + H -14.92900 -16.26000 -9.28900 + H -13.56900 -16.42500 -8.64400 + O -13.77900 -11.57700 -18.76400 + H -14.47500 -12.11600 -19.13900 + H -14.22400 -11.01800 -18.12800 + O -5.68700 -11.57800 -11.11100 + H -5.15400 -10.81000 -11.31700 + H -6.45200 -11.49900 -11.68100 + O -10.12400 -19.07900 -19.08000 + H -10.90200 -18.92600 -19.61700 + H -9.57100 -18.31300 -19.23500 + O -12.14700 -9.82800 -7.04100 + H -11.71600 -10.33900 -6.35600 + H -12.08900 -8.92200 -6.73700 + O -8.56300 -16.78100 -19.39500 + H -7.68900 -17.12900 -19.57200 + H -8.48600 -15.84200 -19.56700 + O -17.82700 -10.80300 -13.33700 + H -17.29000 -11.02400 -14.09700 + H -17.47400 -11.34000 -12.62800 + O -11.81500 -9.78800 -19.46800 + H -12.23500 -9.01000 -19.83700 + H -12.53900 -10.38400 -19.27800 + O -19.86700 -14.07900 -6.37600 + H -19.73600 -14.75100 -5.70600 + H -20.77000 -14.20500 -6.66800 + O -13.47500 -12.64800 -5.70100 + H -14.04000 -12.91200 -6.42700 + H -12.68900 -13.18500 -5.80100 + O -9.69200 -7.27300 -7.13600 + H -10.62300 -7.06600 -7.21300 + H -9.45300 -6.98600 -6.25500 + O -4.27600 -14.80600 -18.47200 + H -4.26900 -15.20000 -19.34400 + H -4.37600 -13.86700 -18.63300 + O -4.39300 -9.46300 -5.94300 + H -4.59100 -8.60700 -5.56400 + H -4.00300 -9.26400 -6.79400 + O -7.99600 -11.65800 -12.76400 + H -8.91400 -11.53100 -12.52200 + H -7.94200 -12.57700 -13.02400 + O -15.18700 -7.64400 -15.48500 + H -15.13500 -6.73900 -15.79200 + H -15.94300 -7.65500 -14.89700 + O -19.71400 -13.10100 -19.77300 + H -19.34400 -12.21800 -19.74800 + H -20.11700 -13.21600 -18.91200 + O -15.18500 -9.90100 -17.07700 + H -15.05400 -9.09200 -16.58300 + H -15.58700 -10.50300 -16.45100 + O -13.63800 -5.48900 -9.10800 + H -13.06600 -4.72800 -9.20600 + H -13.16900 -6.06600 -8.50600 + O -15.15700 -5.03900 -16.46600 + H -16.03200 -4.66000 -16.37800 + H -14.57400 -4.39100 -16.06900 + O -10.86900 -9.42000 -9.46000 + H -11.45400 -9.53200 -8.71100 + H -10.00700 -9.67500 -9.13100 + O -8.93700 -10.02700 -19.59800 + H -8.42800 -10.01800 -18.78800 + H -9.84100 -9.89100 -19.31400 + O -7.27600 -6.37400 -8.20100 + H -8.16200 -6.59900 -7.91700 + H -7.37600 -5.54100 -8.66300 + O -18.98900 -17.52300 -7.36600 + H -19.47100 -18.18500 -7.86200 + H -18.91200 -17.89200 -6.48500 + O -6.93600 -10.22700 -17.64800 + H -7.19800 -11.13500 -17.49900 + H -5.98200 -10.26200 -17.72600 + O -18.97700 -6.57500 -8.98500 + H -18.74900 -7.37200 -8.50600 + H -18.17700 -6.04900 -8.97000 + O -14.76700 -9.77600 -21.77500 + H -14.08800 -9.13200 -21.57200 + H -15.44700 -9.62200 -21.12000 + O -16.23200 -8.99000 -19.46400 + H -15.88500 -9.38200 -18.66200 + H -16.00400 -8.06300 -19.39600 + O -18.53000 -10.59900 -18.85500 + H -19.03100 -10.00600 -18.29400 + H -17.83600 -10.05200 -19.22300 + O -8.83900 -18.99100 -9.42900 + H -9.11900 -18.34400 -8.78300 + H -9.30700 -19.79100 -9.18700 + O -9.49600 -19.74800 -16.43900 + H -9.77500 -19.49700 -17.31900 + H -8.94800 -19.02100 -16.14500 + O -13.25500 -7.71800 -20.65300 + H -13.83000 -7.29100 -20.01800 + H -13.07300 -7.04100 -21.30500 + O -12.75000 -17.35300 -16.77700 + H -11.96900 -16.84100 -16.98800 + H -12.66000 -17.55400 -15.84500 + O -11.97500 -15.64200 -8.26200 + H -11.88500 -15.15200 -7.44400 + H -11.27700 -16.29600 -8.22900 + O -12.18300 -20.49500 -15.78000 + H -12.32800 -19.75400 -15.19200 + H -11.24400 -20.47600 -15.96300 + O -15.10000 -6.44200 -18.96400 + H -15.10900 -5.61400 -19.44400 + H -15.10600 -6.18600 -18.04200 + O -16.13000 -19.26800 -18.71400 + H -16.27400 -18.54700 -18.10100 + H -15.27900 -19.62800 -18.46600 + O -6.99700 -17.57300 -6.59800 + H -7.87200 -17.59300 -6.98600 + H -6.40900 -17.44100 -7.34200 + O -7.86300 -14.32300 -13.41200 + H -8.54700 -14.85900 -13.81400 + H -7.59400 -14.81800 -12.63800 + O -18.66200 -15.74400 -19.73000 + H -18.75800 -14.79900 -19.61200 + H -19.30800 -15.97000 -20.39900 + O -5.86700 -5.58700 -11.48400 + H -5.77900 -5.08800 -12.29600 + H -5.06000 -5.40600 -11.00200 + O -19.78700 -20.31200 -12.76500 + H -19.84600 -20.51100 -13.69900 + H -20.60100 -20.65400 -12.39500 + O -19.63900 -6.20500 -15.86500 + H -19.62100 -6.11800 -14.91200 + H -20.55400 -6.39500 -16.07200 + O -19.97600 -3.77000 -4.55900 + H -19.44500 -3.92100 -3.77700 + H -19.89900 -4.58100 -5.06200 + O -16.36500 -6.12700 -3.54400 + H -15.63800 -6.71800 -3.34900 + H -16.51300 -6.23100 -4.48400 + O -12.95900 -5.62700 21.39600 + H -13.66200 -5.79500 20.76900 + H -12.56100 -4.80900 21.09900 + O -8.98000 -6.50300 -4.62200 + H -9.30600 -7.27200 -4.15400 + H -8.06100 -6.43500 -4.36200 + O -6.70500 -6.21400 -20.70200 + H -6.97300 -7.08800 -20.98400 + H -6.12500 -5.90500 -21.39900 + O -20.47400 -10.76900 -5.19400 + H -19.86200 -11.37600 -4.77700 + H -20.34800 -10.90900 -6.13300 + O -15.45200 -12.39500 21.62500 + H -15.75900 -12.66800 22.48900 + H -15.20400 -11.47700 21.73800 + O -14.64400 -10.19100 -5.02300 + H -14.50300 -9.58300 -5.74800 + H -14.15100 -10.97500 -5.26600 + O -10.98700 -11.16600 -21.76000 + H -11.22400 -10.64100 -20.99500 + H -10.93100 -12.06200 -21.42800 + O -8.57400 -9.65300 -7.97500 + H -7.69500 -9.43100 -8.28300 + H -8.91400 -8.83500 -7.61100 + O -18.88700 -12.25700 -3.47900 + H -17.98100 -12.06300 -3.24200 + H -19.29400 -12.54100 -2.66000 + O -15.80500 -13.23800 -19.63200 + H -16.43700 -13.17900 -18.91500 + H -15.71900 -14.17700 -19.80000 + O -15.45900 -13.51100 -7.49800 + H -16.29500 -13.11500 -7.25200 + H -15.00600 -12.82800 -7.99200 + O -13.59800 -14.46300 21.75600 + H -14.23500 -13.75800 21.63800 + H -12.89900 -14.07000 22.27800 + O -10.70500 -11.16100 -4.99600 + H -9.96100 -11.56000 -5.44700 + H -10.98000 -11.82000 -4.35900 + O -5.45300 -10.61300 -21.21800 + H -4.79800 -10.57500 -21.91500 + H -5.95300 -9.80200 -21.31500 + O -15.16200 -15.87900 -6.19400 + H -15.01700 -16.45500 -6.94400 + H -15.28600 -15.01100 -6.57800 + O -15.40000 -19.37100 -8.34000 + H -14.99900 -18.51200 -8.47100 + H -16.20500 -19.34100 -8.85700 + O -14.00500 -11.45300 -8.43000 + H -13.35200 -10.88700 -8.01900 + H -14.20000 -11.02600 -9.26400 + O -9.02900 -14.29400 -4.23500 + H -9.32500 -14.70800 -3.42400 + H -8.62000 -13.47700 -3.95200 + O -8.51900 -11.98400 -6.53500 + H -8.56400 -11.20900 -7.09600 + H -7.62300 -11.98300 -6.19600 + O -16.74300 -15.86700 -3.92200 + H -17.66800 -15.75500 -4.14400 + H -16.28900 -15.83100 -4.76300 + O -13.11600 -16.46400 -4.40200 + H -13.61300 -16.79100 -3.65200 + H -13.78200 -16.24400 -5.05400 + O -10.62800 -17.51900 -5.20800 + H -11.47000 -17.31400 -4.80200 + H -10.15800 -18.02500 -4.54400 + O -9.07800 -18.94900 -3.37500 + H -9.01300 -19.90300 -3.40200 + H -8.17300 -18.64700 -3.30400 + O -19.56800 -15.99800 -4.32800 + H -19.44200 -16.93500 -4.47300 + H -19.91200 -15.93600 -3.43700 + O -13.84200 -20.96900 -4.25700 + H -13.37000 -21.78100 -4.07300 + H -13.43800 -20.32000 -3.68100 + O -12.91800 -19.21800 -2.27900 + H -13.01100 -19.36400 -1.33800 + H -13.65000 -18.64400 -2.50600 + O -14.78400 -3.94900 -19.98400 + H -13.84200 -3.79600 -20.05100 + H -15.16000 -3.07900 -19.85000 + O -19.86200 -5.80600 -13.17200 + H -20.08200 -6.45000 -12.49900 + H -19.80800 -4.97500 -12.70100 + O -3.84600 -5.27600 -9.59800 + H -4.17100 -5.57200 -8.74800 + H -3.38700 -4.45800 -9.40700 + O -4.86700 -5.75400 -7.03400 + H -5.75000 -5.99800 -7.31300 + H -4.75000 -6.19800 -6.19400 + O -4.12400 -10.18900 -17.44500 + H -3.32000 -9.67400 -17.51100 + H -4.06500 -10.61800 -16.59200 + O -4.49000 -11.79400 -15.22400 + H -5.11100 -12.52200 -15.26200 + H -4.87600 -11.18200 -14.59700 + O -3.44300 -9.80900 -10.96500 + H -2.75200 -10.37900 -11.30400 + H -3.12700 -8.92200 -11.13500 + O -6.11500 -8.80400 -8.87900 + H -6.32800 -7.91700 -8.59100 + H -6.07300 -8.74400 -9.83300 + O -4.16800 -12.16700 -19.29600 + H -4.64200 -11.77500 -20.02900 + H -4.13000 -11.47400 -18.63700 + O -4.83200 -15.25800 -21.34800 + H -5.05700 -15.28700 -22.27800 + H -5.32500 -15.98000 -20.95900 + O -6.12400 -14.00800 -15.48100 + H -5.87200 -14.89200 -15.74500 + H -6.66400 -14.13500 -14.70100 + O -21.04500 -12.36700 -10.39900 + H -20.98400 -11.66200 -11.04300 + H -20.30100 -12.93600 -10.59400 + O -5.91100 -11.78700 -5.38800 + H -5.32000 -12.46500 -5.71500 + H -5.45100 -10.96500 -5.55700 + O -4.24600 -14.18600 -5.81100 + H -5.08500 -14.50900 -5.48300 + H -3.75900 -14.97600 -6.04600 + O -6.01300 -17.60800 -20.31800 + H -5.13100 -17.89600 -20.08000 + H -6.31900 -18.26500 -20.94400 + O -5.47500 -16.53700 -16.56700 + H -5.06500 -17.39800 -16.64300 + H -5.10600 -16.03000 -17.29000 + O -6.68500 -15.61300 -11.24800 + H -5.76200 -15.55700 -11.49500 + H -6.73000 -15.19300 -10.38900 + O -6.47300 -14.34300 -8.73400 + H -7.37600 -14.35800 -8.41900 + H -6.16800 -13.45400 -8.54700 + O 21.30600 -14.17300 -4.31900 + H 22.17700 -14.44400 -4.02700 + H 21.31200 -14.32300 -5.26400 + O -2.25400 -20.19900 -5.38000 + H -2.41800 -21.05700 -5.77100 + H -3.11600 -19.78600 -5.33800 + O -4.63200 -19.31400 -16.71500 + H -4.59500 -19.74700 -15.86300 + H -5.46400 -19.60000 -17.09400 + O -19.83000 -17.54900 -12.64800 + H -19.86400 -18.50500 -12.62100 + H -18.97000 -17.32900 -12.28900 + O -21.80200 -15.56800 -12.45400 + H -21.28300 -15.00500 -13.02900 + H -21.25800 -16.34500 -12.32400 + O -4.64000 -18.87500 -5.74300 + H -5.50600 -18.50700 -5.56400 + H -4.80200 -19.57100 -6.37900 + O -3.36900 -18.45600 -20.31200 + H -3.08700 -19.36800 -20.24200 + H -2.63000 -17.94900 -19.97600 + O -21.14700 -19.75800 -17.60100 + H -21.34600 -20.02400 -18.49900 + H -20.63100 -18.95700 -17.69800 + O -5.21400 -5.78700 -15.88200 + H -5.55100 -6.66500 -15.70500 + H -5.54400 -5.25300 -15.16000 + O -20.27300 -19.46800 -8.91800 + H -20.33800 -20.42100 -8.98600 + H -21.16200 -19.15500 -9.08300 + O -19.76900 -6.25800 -5.77700 + H -20.50600 -6.36000 -6.37900 + H -19.87500 -6.96900 -5.14500 + O -16.40000 -5.35200 -8.72000 + H -16.53800 -4.40500 -8.72700 + H -15.45700 -5.45800 -8.84700 + O -12.71900 -4.77100 -5.01800 + H -12.64300 -5.41900 -4.31800 + H -11.86000 -4.35000 -5.05000 + O -10.12200 -3.91700 -5.01700 + H -9.73800 -4.79300 -5.04500 + H -9.62100 -3.45700 -4.34400 + O -5.03700 -20.76300 -7.67900 + H -5.52500 -20.41100 -8.42300 + H -5.38700 -21.64600 -7.55900 + O -18.78000 -0.79800 -8.11200 + H -19.54200 -1.25000 -8.47300 + H -18.83300 -0.95300 -7.16900 + O -16.44100 -2.57800 -8.71800 + H -17.17000 -1.98400 -8.53800 + H -16.13900 -2.32900 -9.59200 + O -11.79800 -3.44900 -9.33700 + H -11.58000 -2.89000 -10.08300 + H -11.62300 -2.90300 -8.57000 + O -9.77600 -5.25300 -9.93800 + H -10.50900 -4.69000 -9.69100 + H -9.00600 -4.68800 -9.87300 + O -7.32100 -4.01100 -9.71100 + H -6.86600 -3.16900 -9.68600 + H -6.86700 -4.50900 -10.39100 + O -20.09900 -3.39000 -11.67800 + H -21.01500 -3.34300 -11.95200 + H -20.09400 -3.02400 -10.79400 + O -16.67500 -5.27100 -13.49100 + H -17.01200 -6.15700 -13.35900 + H -15.95800 -5.19100 -12.86100 + O -14.44900 -5.44300 -11.78100 + H -14.27800 -6.31600 -12.13200 + H -14.17900 -5.49500 -10.86400 + O -12.30700 -20.20000 -12.17900 + H -13.15800 -20.63200 -12.25700 + H -11.67600 -20.91900 -12.13500 + O -6.40200 -22.12800 -13.07200 + H -5.70800 -21.66200 -13.53800 + H -7.20900 -21.69000 -13.34300 + O -4.30800 -20.78700 -14.30000 + H -3.69900 -21.49300 -14.08300 + H -3.84800 -19.98700 -14.04700 + O -19.57700 -20.71700 -15.50200 + H -20.16400 -20.46800 -16.21700 + H -19.52200 -21.67100 -15.55800 + O -20.33900 -21.97500 -20.80400 + H -19.49900 -21.60000 -20.53800 + H -20.97300 -21.27400 -20.65600 + O -14.92400 -21.00500 -15.34300 + H -14.01100 -20.85500 -15.59000 + H -14.89100 -21.19400 -14.40600 + O -8.86200 -21.42400 -14.26200 + H -9.10700 -20.74600 -14.89100 + H -8.63900 -22.18300 -14.80100 + O -6.94200 -20.55600 -17.49200 + H -7.82600 -20.50300 -17.13000 + H -6.77100 -21.49500 -17.57100 + O -17.99200 -20.65700 -20.30200 + H -17.50200 -20.58900 -21.12100 + H -17.42600 -20.24800 -19.64800 + O -13.46800 -19.65300 -18.08100 + H -13.06900 -20.29700 -17.49600 + H -13.25300 -18.80700 -17.68900 + O -11.96400 -4.62100 -20.12100 + H -11.01800 -4.67900 -20.25100 + H -12.33900 -5.01000 -20.91100 + O -18.93700 -18.72800 -4.75800 + H -19.58200 -19.41300 -4.58300 + H -18.09500 -19.13300 -4.55000 + O -16.66500 -20.39000 -4.01800 + H -15.70800 -20.40600 -4.02600 + H -16.92100 -21.31100 -4.07300 + O -7.97900 -21.53000 -3.16700 + H -8.47300 -22.19400 -2.68700 + H -7.47100 -22.02600 -3.80900 + O 8.46000 -12.19900 -11.83800 + H 8.63500 -12.68500 -11.03200 + H 8.02800 -12.83200 -12.41100 + O 0.30400 -13.22700 -10.11100 + H 0.01600 -14.07400 -9.76800 + H 0.30300 -12.64700 -9.34900 + O 12.19500 -9.50600 -14.48500 + H 11.92800 -10.08400 -13.77000 + H 12.35400 -8.66100 -14.06400 + O 3.21100 -9.01700 -13.09500 + H 3.05600 -9.88100 -12.71200 + H 4.11900 -8.81500 -12.87100 + O -2.83000 -14.07900 -16.03200 + H -3.26800 -13.25400 -15.82100 + H -3.27200 -14.39100 -16.82100 + O -2.07500 -14.03100 -13.30400 + H -1.31300 -14.60200 -13.20200 + H -2.26700 -14.05300 -14.24200 + O 8.69400 -5.25000 -12.09300 + H 9.34900 -5.80000 -11.66400 + H 8.15900 -4.91000 -11.37500 + O 7.20300 -9.87600 -10.89900 + H 7.51600 -10.68600 -11.30300 + H 7.94700 -9.56100 -10.38600 + O 15.10000 -18.90500 -10.81300 + H 15.10900 -18.19600 -10.17000 + H 15.90200 -18.78000 -11.32000 + O 9.17500 -15.24800 -12.43700 + H 9.58600 -16.07200 -12.69900 + H 8.31200 -15.26800 -12.85000 + O 2.87500 -17.13300 -13.23200 + H 2.59000 -16.55100 -13.93600 + H 2.48500 -17.98000 -13.44700 + O 10.12100 -17.37800 -15.84800 + H 10.30900 -17.43400 -14.91100 + H 10.91900 -17.01800 -16.23300 + O -0.02100 -15.92300 -12.79500 + H -0.42500 -16.64300 -12.31100 + H -0.14800 -16.15300 -13.71600 + O 4.32700 -13.72900 -13.24500 + H 5.18900 -14.13400 -13.34200 + H 4.05500 -13.52800 -14.14100 + O 15.08800 -17.02000 -8.82400 + H 14.34100 -16.43000 -8.92800 + H 15.13400 -17.18500 -7.88200 + O 0.71100 -18.56700 -9.74900 + H 0.10100 -18.24500 -10.41300 + H 0.23700 -18.47700 -8.92300 + O 8.96500 -8.62600 -16.19800 + H 8.53000 -8.20500 -16.93900 + H 8.64200 -8.15400 -15.43000 + O 4.72200 -10.69700 -9.68200 + H 4.14600 -9.96700 -9.45500 + H 5.53700 -10.28100 -9.96200 + O 10.80000 -17.35400 -13.17500 + H 10.58500 -18.10400 -12.62100 + H 11.65100 -17.05600 -12.85600 + O 2.80700 -8.80800 -8.82300 + H 2.89900 -7.85500 -8.79900 + H 1.86300 -8.95900 -8.76400 + O -1.71500 -8.80100 -17.87000 + H -1.56200 -8.38500 -18.71900 + H -1.47800 -8.13100 -17.22900 + O 12.33500 -7.23700 -12.84000 + H 11.64700 -7.14300 -12.18100 + H 12.91900 -6.49500 -12.68600 + O 12.40900 -19.23900 -9.90500 + H 11.73500 -19.14600 -10.57800 + H 13.23500 -19.12300 -10.37500 + O 4.17900 -21.04700 -11.74900 + H 4.43100 -21.58500 -10.99900 + H 3.24700 -20.87200 -11.61800 + O 13.18300 -8.81200 -8.45000 + H 12.94600 -9.33600 -9.21500 + H 13.37300 -7.94300 -8.80400 + O 7.25200 -15.39400 -21.76400 + H 8.04600 -15.91300 -21.63700 + H 7.47800 -14.77500 -22.45800 + O 14.04800 -12.76000 -9.95500 + H 13.47600 -11.99700 -10.04400 + H 14.60900 -12.73200 -10.72900 + O 10.89300 -14.27200 -7.92100 + H 10.81100 -13.52300 -7.33100 + H 10.15700 -14.18300 -8.52700 + O 3.28700 -13.40000 -15.76500 + H 2.87500 -13.07300 -16.56500 + H 2.82400 -14.21500 -15.57200 + O 5.81600 -8.19700 -12.60100 + H 6.46800 -7.89300 -13.23200 + H 6.30600 -8.76100 -12.00300 + O 10.40900 -7.01700 -10.72700 + H 10.73400 -6.65300 -9.90300 + H 10.09500 -7.89100 -10.49500 + O 4.17400 -9.06100 -6.41900 + H 3.58300 -8.46300 -5.96200 + H 3.75900 -9.20600 -7.26900 + O 0.02200 -12.13800 -7.54600 + H 0.20300 -11.46100 -6.89500 + H -0.92400 -12.27800 -7.49000 + O 7.72300 -6.88300 -14.20700 + H 8.18600 -6.36900 -13.54500 + H 7.11100 -6.26500 -14.60700 + O 2.17200 -7.17100 -5.96400 + H 2.49100 -6.68900 -6.72700 + H 1.25500 -7.35800 -6.16400 + O 14.25700 -20.66700 -20.73500 + H 14.36500 -19.72900 -20.58100 + H 15.05900 -21.06100 -20.39200 + O 1.94100 -12.92600 -18.09600 + H 1.17900 -12.36600 -18.24400 + H 1.69700 -13.76900 -18.47700 + O -2.59900 -9.71800 -8.23400 + H -3.02800 -9.69400 -9.09000 + H -2.65400 -10.63500 -7.96400 + O -2.47600 -7.18100 -11.05400 + H -1.63500 -7.04900 -10.61600 + H -3.08000 -6.59900 -10.59100 + O 15.22800 -7.28600 -5.76600 + H 15.16300 -7.97200 -5.10100 + H 14.35300 -6.89800 -5.79600 + O 6.57700 -7.83300 -7.04700 + H 6.49000 -6.95900 -6.66600 + H 5.77200 -8.28600 -6.79500 + O 7.77900 -4.07500 -17.08800 + H 8.13500 -4.25500 -17.95800 + H 7.81400 -3.12200 -17.00600 + O 1.21500 -5.47100 -15.42000 + H 1.45600 -5.84500 -14.57200 + H 0.52700 -6.04800 -15.75000 + O 11.03200 -3.89400 -20.43100 + H 11.67900 -4.45400 -20.85900 + H 11.36200 -3.00400 -20.55700 + O 12.52000 -15.70500 -6.18900 + H 11.88400 -15.24000 -6.73200 + H 12.52800 -16.59600 -6.53700 + O -0.01600 -8.59600 -7.82700 + H 0.22700 -9.12000 -7.06400 + H -0.91000 -8.87000 -8.03200 + O 1.47200 -20.76900 -11.28300 + H 0.71200 -20.69100 -11.85900 + H 1.34500 -20.08800 -10.62300 + O 6.86700 -20.27400 -8.30900 + H 7.04000 -21.17300 -8.03100 + H 5.95100 -20.12200 -8.07600 + O 9.61700 -17.84200 -6.88900 + H 9.41100 -18.22400 -6.03600 + H 9.16600 -16.99800 -6.89000 + O 9.69900 -16.71800 21.51800 + H 10.55000 -16.89800 21.91700 + H 9.90900 -16.32500 20.67100 + O 6.27600 -19.40700 -20.42900 + H 5.60000 -18.87700 -20.85200 + H 5.96800 -19.51700 -19.53000 + O 1.86000 -15.72300 -15.36400 + H 0.90900 -15.75500 -15.46800 + H 2.19500 -16.31200 -16.04000 + O 8.00800 -15.62100 -6.50900 + H 7.17600 -15.81400 -6.07700 + H 8.59000 -15.35300 -5.79800 + O 7.98500 -15.63500 -16.17100 + H 8.41500 -14.84000 -16.48500 + H 8.68600 -16.28500 -16.12200 + O 0.17900 -15.65600 -8.83500 + H 0.78500 -15.26500 -8.20600 + H -0.21400 -16.39200 -8.36400 + O 4.15900 -18.19300 -21.74400 + H 3.39000 -18.75800 -21.82200 + H 3.94100 -17.41700 -22.25800 + O -2.90100 -18.45800 -14.07500 + H -2.31700 -18.08000 -13.41800 + H -2.49900 -18.23200 -14.91400 + O -1.03800 -17.45600 -19.24400 + H -0.39900 -18.14200 -19.43400 + H -0.70400 -16.68400 -19.70300 + O 8.89300 -14.17800 -9.91200 + H 9.13000 -14.65000 -10.71000 + H 8.14100 -14.65700 -9.56400 + O 5.54900 -17.55400 -13.77900 + H 4.65100 -17.49500 -13.45200 + H 5.87300 -18.38800 -13.44000 + O -0.89200 -16.04900 -15.47700 + H -1.48800 -15.35200 -15.75100 + H -1.17400 -16.81500 -15.97700 + O 2.95400 -17.17300 -17.38800 + H 3.89300 -17.06800 -17.24000 + H 2.76300 -16.60200 -18.13300 + O 10.03900 -19.31800 -11.41000 + H 9.47200 -18.94800 -10.73300 + H 9.61400 -20.14000 -11.65600 + O 2.29100 -15.42100 -19.42100 + H 1.51700 -15.45500 -19.98200 + H 3.02800 -15.37300 -20.03100 + O 13.41500 -3.44900 -5.45400 + H 13.87900 -3.36600 -6.28700 + H 14.05300 -3.17700 -4.79500 + O 0.44900 -10.19400 -5.59700 + H 1.05600 -10.44900 -4.90200 + H -0.39500 -10.09900 -5.15600 + O 6.74000 -15.09900 -13.75800 + H 7.11200 -15.12500 -14.64000 + H 6.31500 -15.94900 -13.65200 + O 1.81700 -19.32200 -14.49100 + H 1.02100 -19.63800 -14.91700 + H 2.44400 -20.03800 -14.59300 + O 4.20500 -15.46000 -9.68400 + H 4.20100 -14.65100 -9.17300 + H 5.11400 -15.75900 -9.65500 + O 5.44700 -10.52800 -18.97600 + H 4.91400 -11.20300 -19.39700 + H 5.36600 -10.70900 -18.03900 + O 12.48000 -10.38500 -10.62600 + H 13.32400 -10.18000 -11.02800 + H 11.91700 -10.62900 -11.36100 + O 8.88600 -20.04000 -19.39900 + H 8.07500 -19.77000 -19.82900 + H 9.56100 -19.93100 -20.06900 + O 8.08100 -10.29900 -7.57300 + H 8.08900 -10.93300 -6.85600 + H 7.44700 -9.63600 -7.29900 + O 3.30900 -11.53800 -11.90900 + H 3.83700 -12.19800 -12.35900 + H 3.79100 -11.35600 -11.10200 + O 7.56900 -9.42100 -20.22300 + H 7.76600 -8.60900 -19.75500 + H 6.79200 -9.76800 -19.78400 + O -2.73600 -12.44600 -7.59900 + H -3.19500 -12.92100 -6.90600 + H -3.13200 -12.76400 -8.41000 + O 5.81600 -13.24000 -6.01700 + H 5.75700 -14.11500 -5.63200 + H 6.74600 -13.01700 -5.96400 + O 9.35200 -7.54100 -6.64300 + H 8.44800 -7.71900 -6.90400 + H 9.33400 -7.56300 -5.68600 + O 15.06900 -10.24500 -7.02100 + H 14.38200 -9.66000 -7.33900 + H 15.69100 -10.30100 -7.74700 + O 10.98000 -11.36100 -12.78400 + H 10.08800 -11.45700 -12.45100 + H 11.30900 -12.25800 -12.85100 + O 3.96000 -8.90900 -15.76500 + H 3.94800 -7.98700 -16.02100 + H 3.56300 -8.92100 -14.89400 + O -1.58100 -13.02900 -19.86100 + H -1.01000 -12.35800 -19.48600 + H -2.45500 -12.80600 -19.54000 + O 5.10900 -11.30500 -16.38900 + H 4.71200 -10.48100 -16.10900 + H 4.51300 -11.98100 -16.06800 + O 5.78200 -6.11700 -9.24100 + H 6.37600 -5.37300 -9.33800 + H 6.24800 -6.72200 -8.66300 + O 3.89900 -6.12500 -16.21000 + H 3.05300 -5.73400 -15.99400 + H 4.53400 -5.61900 -15.70200 + O 9.70700 -9.48200 -9.64100 + H 9.25400 -9.78800 -8.85600 + H 10.54800 -9.93900 -9.62600 + O 11.09000 -6.05200 -8.11400 + H 10.47400 -6.59200 -7.62100 + H 10.61400 -5.23900 -8.28500 + O -0.71900 -17.92400 -7.40000 + H -1.38400 -18.55800 -7.66900 + H -0.37700 -18.27100 -6.57600 + O -0.32500 -6.25100 -9.42600 + H -0.15400 -6.98800 -8.83900 + H -0.31900 -5.48300 -8.85400 + O 4.27900 -8.84600 21.09400 + H 5.12500 -9.29100 21.13600 + H 4.07000 -8.64100 22.00600 + O 3.98400 -8.41800 -20.05200 + H 4.44500 -9.14500 -19.63300 + H 4.35900 -7.63300 -19.65200 + O -0.11000 -11.03900 -18.63900 + H -0.67200 -10.42700 -18.16400 + H 0.44900 -10.48200 -19.18000 + O 8.47000 -18.36600 -9.34100 + H 9.01000 -18.23200 -8.56200 + H 7.86400 -19.06500 -9.09500 + O 9.37300 -19.96100 -16.64500 + H 9.15300 -19.94100 -17.57700 + H 9.56200 -19.04900 -16.42400 + O 7.85800 -7.27200 -18.41000 + H 6.91700 -7.10100 -18.37600 + H 8.24000 -6.45100 -18.72200 + O 5.63600 -17.10600 -16.51600 + H 6.41700 -16.55700 -16.58300 + H 5.54900 -17.28700 -15.58000 + O 6.94800 -16.01000 -9.08400 + H 7.21500 -15.85900 -8.17700 + H 7.36400 -16.83800 -9.32300 + O 6.80700 -20.73600 -15.64300 + H 6.76600 -20.47000 -14.72400 + H 7.71300 -20.56700 -15.90100 + O 5.23200 -6.38200 -18.61300 + H 5.28200 -5.48300 -18.93800 + H 4.73400 -6.31500 -17.79800 + O 2.69900 -19.91600 -17.98700 + H 2.51300 -19.07600 -17.56900 + H 3.64700 -19.91600 -18.11700 + O 12.30800 -18.16000 -7.35800 + H 11.36200 -18.16900 -7.21000 + H 12.41800 -18.53800 -8.23000 + O 11.59400 -14.02600 -13.01200 + H 10.74400 -14.40800 -12.79100 + H 12.23300 -14.67300 -12.71100 + O 1.31300 -9.25800 -20.24000 + H 2.16500 -8.83600 -20.13100 + H 1.47800 -9.97900 -20.84700 + O -0.14000 -15.24700 -20.73000 + H -0.63600 -14.50100 -20.39300 + H -0.19500 -15.16000 -21.68200 + O 14.00500 -4.95100 -11.88700 + H 14.21000 -4.06400 -12.18400 + H 14.85900 -5.36400 -11.75800 + O -0.85000 -20.58800 -12.77300 + H -0.66400 -20.41600 -13.69600 + H -1.42700 -21.35100 -12.78200 + O -1.27900 -6.57500 -16.20800 + H -1.65400 -6.46400 -15.33500 + H -1.66700 -5.87100 -16.72800 + O -1.01400 -3.92200 -3.90800 + H -0.29800 -4.13400 -3.30900 + H -1.40700 -4.76800 -4.12100 + O 2.75600 -6.56200 -3.32600 + H 3.48800 -7.15700 -3.16200 + H 2.50800 -6.73100 -4.23400 + O 7.58400 -5.81000 21.91000 + H 6.63200 -5.74000 21.83600 + H 7.92000 -5.28000 21.18800 + O 9.29300 -7.69900 -3.90500 + H 9.40300 -8.50600 -3.40200 + H 10.02100 -7.14200 -3.62900 + O 12.93300 -5.51100 -21.67000 + H 13.33100 -6.21100 -21.15100 + H 13.65000 -5.16900 -22.20300 + O -2.10000 -10.10400 -4.51600 + H -2.14700 -11.05800 -4.44800 + H -2.87000 -9.86200 -5.03000 + O 2.01000 -11.50700 -21.59900 + H 2.79700 -11.83600 -21.16400 + H 1.97400 -11.98800 -22.42600 + O 4.79600 -11.05400 -4.62400 + H 4.65800 -10.36000 -5.26900 + H 5.07800 -11.80900 -5.13900 + O 6.81300 -9.99000 21.08700 + H 7.03500 -10.03800 22.01700 + H 7.45100 -9.37900 20.71900 + O 11.17300 -9.69300 -6.78900 + H 11.91700 -9.31800 -7.25900 + H 10.52700 -8.98700 -6.75800 + O -2.34600 -12.77700 -4.35200 + H -3.00800 -13.35900 -4.72600 + H -2.33900 -12.99000 -3.41900 + O 3.98900 -12.57600 -19.93000 + H 3.34300 -12.75800 -19.24800 + H 4.20500 -13.43500 -20.29200 + O 1.92300 -14.12500 -7.33900 + H 1.27000 -13.42900 -7.27300 + H 2.71200 -13.68400 -7.65300 + O 4.51900 -15.00400 -21.06500 + H 4.14600 -15.25300 -21.91100 + H 5.45600 -15.17500 -21.15600 + O 8.38800 -12.27200 -5.48700 + H 9.32200 -12.31500 -5.69400 + H 8.33700 -12.50500 -4.56000 + O 3.00500 -16.28900 -6.05000 + H 2.96700 -16.92900 -6.76100 + H 2.58400 -15.50800 -6.40900 + O 3.01300 -17.71300 -8.45900 + H 3.40800 -16.99700 -8.95600 + H 2.25500 -17.97700 -8.98000 + O 4.27600 -13.07000 -8.31300 + H 4.85900 -13.06900 -7.55400 + H 4.46200 -12.24500 -8.76000 + O 9.48200 -15.65900 -4.08700 + H 9.18300 -16.56800 -4.08800 + H 10.40200 -15.70900 -3.82800 + O 11.16200 -12.24700 -5.91000 + H 11.11400 -11.33900 -6.20900 + H 12.10000 -12.43300 -5.85600 + O 2.50200 -16.14100 -3.33100 + H 1.56400 -16.27800 -3.19500 + H 2.61500 -16.17600 -4.28100 + O 4.15000 -20.13700 -7.75700 + H 3.78000 -19.29400 -8.01900 + H 3.76400 -20.31200 -6.89900 + O 5.65400 -16.11800 -5.25200 + H 5.81700 -16.92700 -4.76700 + H 4.72700 -16.16100 -5.48600 + O 8.78700 -18.32900 -4.29000 + H 7.86200 -18.54100 -4.17200 + H 9.24500 -18.89300 -3.66700 + O 10.28800 -19.80900 -2.45300 + H 10.35700 -20.75700 -2.34600 + H 10.06700 -19.48500 -1.57900 + O -0.08800 -17.02500 -2.87700 + H -0.18900 -17.51900 -3.69100 + H 0.05200 -17.69200 -2.20600 + O 5.71300 -21.40200 -4.62800 + H 6.17000 -22.06300 -4.10700 + H 5.11300 -21.90500 -5.17900 + O 6.00600 -18.73500 -3.99300 + H 5.94000 -19.63700 -4.30600 + H 5.22100 -18.60800 -3.46100 + O 5.71700 -3.81300 -19.67500 + H 6.66900 -3.91000 -19.65200 + H 5.56000 -2.91100 -19.39900 + O -0.93100 -7.64600 -20.37900 + H -0.18600 -8.22900 -20.23600 + H -0.78300 -7.27400 -21.24800 + O -2.85000 -3.70500 -15.16000 + H -2.99300 -4.08100 -16.02800 + H -2.79400 -4.46300 -14.57700 + O -2.09800 -5.90100 -13.60000 + H -2.49800 -6.51800 -12.98700 + H -1.37100 -5.51400 -13.11200 + O 16.48600 -5.90500 -11.14000 + H 16.56600 -5.89400 -10.18600 + H 17.35300 -5.65000 -11.45600 + O 15.98100 -6.10000 -8.22100 + H 15.15600 -6.20500 -8.69400 + H 15.87400 -6.62800 -7.43000 + O 14.55900 -8.76600 -3.41600 + H 13.68100 -8.72600 -3.03500 + H 15.12400 -8.36100 -2.75800 + O 15.01800 -10.20000 -11.55800 + H 15.40400 -11.01300 -11.88300 + H 15.55400 -9.50800 -11.94600 + O 13.53500 -6.24900 -9.42200 + H 12.69800 -5.97600 -9.04700 + H 13.58800 -5.78300 -10.25600 + O 11.95300 -13.58000 -15.72700 + H 12.17200 -14.43600 -16.09400 + H 11.93500 -13.72200 -14.78100 + O -1.66400 -11.86900 -11.61800 + H -1.81600 -12.48500 -12.33500 + H -0.86200 -12.18100 -11.20000 + O 13.91900 -12.38000 -5.73600 + H 14.46600 -13.14600 -5.91200 + H 14.33200 -11.67200 -6.22900 + O 15.08300 -14.79400 -6.44200 + H 14.16000 -15.02600 -6.34200 + H 15.55400 -15.61100 -6.27900 + O 12.28600 -17.64200 -21.59100 + H 13.12300 -17.70300 -21.13100 + H 11.88100 -18.50000 -21.46500 + O 12.57600 -16.36700 -16.70500 + H 13.12400 -17.05400 -16.32500 + H 12.80300 -16.36600 -17.63500 + O 13.08800 -16.18300 -12.05600 + H 14.00800 -16.14200 -12.31400 + H 13.09300 -16.00100 -11.11600 + O 13.03400 -15.24500 -9.42700 + H 12.24900 -15.06300 -8.91000 + H 13.42100 -14.38500 -9.58700 + O -3.14400 -16.63200 -6.46900 + H -2.26100 -16.91000 -6.71300 + H -3.54300 -17.41200 -6.08400 + O 16.44800 -20.98500 -4.26700 + H 16.53200 -21.86600 -4.63200 + H 15.73300 -20.58700 -4.76200 + O 13.97600 -18.73800 -16.09500 + H 14.24000 -19.04400 -15.22700 + H 13.33400 -19.38300 -16.39200 + O -1.32400 -18.11300 -11.68200 + H -1.20500 -19.02000 -11.96300 + H -2.15700 -18.11600 -11.20900 + O -3.96100 -15.32300 -11.68700 + H -3.43000 -14.97100 -12.40200 + H -3.85600 -14.69100 -10.97600 + O 14.39900 -19.87200 -5.66600 + H 13.61400 -19.47300 -5.29000 + H 14.06700 -20.46700 -6.33800 + O 14.92000 -17.97200 -20.52700 + H 15.31400 -17.95400 -21.39900 + H 15.63500 -17.73300 -19.93600 + O -2.04100 -18.24600 -16.78800 + H -2.92500 -18.58500 -16.92600 + H -1.74000 -17.99600 -17.66100 + O -2.69300 -19.42300 -8.54700 + H -3.32900 -20.02000 -8.15300 + H -3.14400 -19.05800 -9.30800 + O -2.16000 -6.34000 -4.26400 + H -3.04600 -6.55200 -4.56000 + H -1.97400 -6.98700 -3.58300 + O 3.19100 -6.09700 -8.30200 + H 2.86200 -5.21400 -8.47000 + H 4.11000 -6.06500 -8.56800 + O 6.55500 -5.21100 -6.11900 + H 6.60700 -5.62400 -5.25700 + H 7.32300 -4.63900 -6.15700 + O 8.71100 -3.49300 -6.36200 + H 9.20500 -3.56200 -7.17900 + H 9.36500 -3.25200 -5.70600 + O 13.74100 22.00300 -7.59500 + H 12.97900 22.12700 -8.16000 + H 13.77900 21.05800 -7.44500 + O -0.26400 -3.75900 -8.01700 + H -1.14000 -3.47200 -8.27600 + H -0.08400 -3.27800 -7.20900 + O 2.37700 -3.51300 -8.79400 + H 1.45800 -3.58000 -8.53300 + H 2.35900 -3.01000 -9.60800 + O 7.16300 -3.78100 -10.09900 + H 6.45100 -3.55800 -10.69900 + H 7.13300 -3.10000 -9.42700 + O 9.76300 -3.83100 -8.93400 + H 8.95000 -3.73600 -9.43100 + H 10.45600 -3.66800 -9.57500 + O 11.68700 -3.70400 -10.87400 + H 11.60800 -2.91100 -11.40500 + H 12.45900 -4.15000 -11.22200 + O -0.00900 -5.12500 -11.94900 + H -0.12500 -5.44900 -11.05600 + H 0.73400 -5.62100 -12.29000 + O 2.11100 -6.50200 -13.00100 + H 2.28000 -7.44200 -13.06600 + H 2.87300 -6.15300 -12.53900 + O 4.48100 -5.94300 -11.67200 + H 4.91900 -6.65400 -12.13900 + H 4.80400 -6.01000 -10.77400 + O 6.44100 -20.07300 -12.96400 + H 5.71000 -20.36100 -12.41800 + H 7.20200 -20.53300 -12.60900 + O 11.83100 21.60600 -12.10600 + H 12.74200 21.57900 -12.40000 + H 11.36000 22.02600 -12.82600 + O 14.31900 21.59100 -13.18700 + H 14.98400 20.93700 -13.40100 + H 14.68200 22.41700 -13.50800 + O -0.54600 -20.32800 -15.50900 + H -1.04400 -19.74900 -16.08600 + H -0.76900 -21.21100 -15.80600 + O -1.92700 -20.82300 -19.94600 + H -1.04700 -20.46400 -19.82800 + H -1.80900 -21.55200 -20.55500 + O 3.83600 -21.24200 -14.48300 + H 4.66500 -21.11900 -14.94600 + H 4.06100 -21.14100 -13.55800 + O 10.28400 -21.63000 -14.45200 + H 10.07800 -21.05300 -15.18700 + H 10.64700 -22.41600 -14.85900 + O 12.14100 -20.49200 -17.13800 + H 11.19300 -20.41000 -17.03400 + H 12.25100 -21.11500 -17.85600 + O -2.38400 -4.53900 -17.84300 + H -2.32900 -4.98400 -18.68800 + H -2.19100 -3.62300 -18.04100 + O 0.59100 -19.61600 -19.79500 + H 1.02800 -19.56200 -20.64400 + H 1.29200 -19.82100 -19.17600 + O 5.46800 -19.54200 -17.79400 + H 5.91000 -20.09200 -17.14700 + H 5.52200 -18.65700 -17.43300 + O 8.53600 -4.76100 -19.66900 + H 9.40300 -4.41900 -19.88600 + H 8.22900 -5.16900 -20.47900 + O 0.05600 -18.94800 -4.89100 + H -0.76900 -19.42200 -5.00200 + H 0.61200 -19.54700 -4.39300 + O 1.22600 -20.89900 -3.25300 + H 1.88200 -21.48500 -3.63200 + H 0.52400 -21.48100 -2.96200 + O 2.00800 -5.12300 -21.65500 + H 1.90000 -4.23100 -21.98600 + H 1.67000 -5.08900 -20.76000 + O 10.73000 21.36200 -2.31700 + H 10.06900 20.68200 -2.44800 + H 11.49600 21.04800 -2.79600 + O 12.74000 -6.08800 -5.79400 + H 12.22300 -6.14400 -6.59700 + H 12.90600 -5.15300 -5.67900 + O 17.78200 -14.66000 -10.94000 + H 17.76800 -15.39400 -10.32500 + H 18.69100 -14.36300 -10.93900 + O 19.50700 -9.31200 -11.76900 + H 19.45300 -10.20200 -12.11700 + H 19.40700 -9.41800 -10.82300 + O 17.33400 -12.86900 -14.24200 + H 17.58100 -13.77800 -14.41000 + H 18.11500 -12.47600 -13.85200 + O 21.01500 -19.10700 -13.65800 + H 21.50600 -18.60700 -14.30900 + H 20.31000 -19.52200 -14.15600 + O 17.97800 -15.58000 -14.45200 + H 18.53100 -15.99100 -13.78700 + H 18.29200 -15.94000 -15.28100 + O 20.97100 -18.89000 -9.41400 + H 20.92800 -17.93500 -9.37100 + H 20.12100 -19.18400 -9.08700 + O 19.66600 -16.91600 -12.70600 + H 20.39300 -16.31600 -12.54200 + H 20.08600 -17.74200 -12.94600 + O 21.80300 -8.72900 -9.02100 + H 21.84900 -7.88500 -8.57300 + H 20.87000 -8.94400 -9.03300 + O 21.25200 -12.84000 -14.50200 + H 21.22000 -13.60300 -15.07900 + H 22.13000 -12.48000 -14.63000 + O 20.57400 -13.94900 -10.88900 + H 21.10100 -14.48400 -11.48300 + H 21.20300 -13.35400 -10.48100 + O 19.09100 -13.09700 -7.69700 + H 19.14100 -12.22300 -7.31100 + H 18.17900 -13.18700 -7.97300 + O 20.67600 -6.75800 -5.86700 + H 21.07900 -6.48800 -6.69200 + H 19.74200 -6.58200 -5.98700 + O 16.65700 -10.55400 -9.26500 + H 16.10700 -10.43700 -10.04000 + H 16.64000 -11.49700 -9.09900 + O 16.84300 -8.39400 -12.40600 + H 17.78900 -8.53200 -12.37900 + H 16.71900 -7.50900 -12.06400 + O 19.08700 -9.40000 -8.86800 + H 19.29900 -9.78500 -8.01800 + H 18.21400 -9.73500 -9.07300 + O 21.58100 -20.67300 -11.48000 + H 21.32800 -20.16100 -12.24800 + H 21.37400 -20.10800 -10.73600 + O 20.46200 -16.16300 -9.13100 + H 19.52100 -16.32900 -9.07200 + H 20.53700 -15.40500 -9.71000 + O 21.37000 -15.11800 -16.12400 + H 20.54300 -15.47300 -16.45100 + H 21.78600 -15.85300 -15.67400 + O 17.79700 -16.84900 -9.22900 + H 16.85400 -16.80400 -9.07100 + H 18.03700 -17.74600 -8.99600 + O 14.97700 -19.58400 -13.62500 + H 15.86900 -19.25500 -13.52000 + H 14.56600 -19.44000 -12.77200 + O 17.47300 -17.69400 -19.06100 + H 17.93500 -18.46400 -19.39200 + H 17.95800 -16.95100 -19.42000 + O 19.10700 -16.76400 -16.71800 + H 18.49400 -16.91800 -17.43700 + H 19.54400 -17.60600 -16.59100 + O 21.72000 -17.00200 -19.55000 + H 20.80400 -16.72700 -19.57800 + H 22.16100 -16.43300 -20.18100 + O 19.33600 -10.62100 -6.42500 + H 19.96700 -10.80700 -5.73000 + H 18.52800 -10.39700 -5.96300 + O 20.19200 -19.28800 -16.59400 + H 19.66900 -20.08500 -16.68100 + H 21.07200 -19.54800 -16.86500 + O 19.52300 -11.96600 -12.62100 + H 20.12900 -12.16300 -13.33500 + H 19.72800 -12.61300 -11.94600 + O 16.27400 -13.15200 -8.31400 + H 16.04200 -13.73700 -7.59300 + H 15.49200 -13.12900 -8.86600 + O 18.40300 -19.41700 -8.38600 + H 17.63600 -19.94600 -8.60700 + H 18.52000 -19.54700 -7.44500 + O 18.92800 -6.62800 -9.29800 + H 18.87300 -7.55000 -9.04700 + H 18.32500 -6.18100 -8.70400 + O 21.87400 -19.70800 -20.28000 + H 21.90500 -18.77700 -20.06100 + H 21.42200 -19.74100 -21.12300 + O 19.20300 -15.81500 -20.11600 + H 19.39200 -14.87600 -20.10200 + H 18.85700 -15.97700 -20.99300 + O 18.60700 -20.51300 -14.04000 + H 18.38300 -20.60100 -14.96600 + H 18.97400 -21.36300 -13.80000 + O 18.02700 -3.88100 -4.64000 + H 18.69500 -4.04400 -3.97400 + H 17.89600 -4.73000 -5.06100 + O 21.44200 -7.33600 -3.33300 + H 22.35700 -7.60700 -3.40700 + H 21.16000 -7.19300 -4.23600 + O 17.00300 -10.23000 -5.05400 + H 16.96100 -11.11200 -4.68400 + H 16.24800 -10.18100 -5.64100 + O 17.06400 -12.70600 -3.94300 + H 17.61400 -13.48900 -3.96200 + H 16.93300 -12.52800 -3.01100 + O 21.34700 -14.53800 -7.03400 + H 20.61700 -13.96100 -7.25900 + H 21.25800 -15.28200 -7.63000 + O 18.80300 -14.94300 -3.39200 + H 18.56400 -15.72200 -3.89300 + H 19.64300 -14.66900 -3.75900 + O 18.04500 -17.43300 -4.42100 + H 18.34800 -18.06100 -5.07600 + H 18.28700 -17.82400 -3.58200 + O 16.03600 -12.71300 -11.85600 + H 16.29600 -12.74000 -12.77700 + H 16.62500 -13.33100 -11.42300 + O 15.76500 -17.45000 -6.11600 + H 16.47200 -17.43500 -5.47100 + H 15.31500 -18.28000 -5.96000 + O 17.38200 -18.68600 -12.40600 + H 17.86600 -19.36100 -12.88100 + H 17.99700 -17.95700 -12.33000 + O 15.78100 -15.81300 -12.61400 + H 16.28100 -15.82900 -13.43000 + H 16.38200 -15.43900 -11.97100 + O 16.16800 -19.70100 -17.57500 + H 15.41100 -19.40200 -17.07300 + H 16.55200 -18.90000 -17.93200 + O 15.97900 -20.77800 -8.83700 + H 15.19200 -21.08000 -8.38300 + H 15.64800 -20.27400 -9.58100 + O 17.86900 -6.48500 -5.49900 + H 16.96700 -6.72000 -5.71900 + H 18.07900 -7.02700 -4.73800 + O -21.58100 -6.16400 -8.04500 + H -22.04200 -5.45900 -8.49900 + H -20.70200 -6.16000 -8.42500 + O 18.36900 -3.59000 -8.80200 + H 17.65100 -3.20900 -9.30700 + H 18.43300 -3.04000 -8.02200 + O 21.04800 -4.11900 -9.33500 + H 20.11700 -4.08700 -9.11500 + H 21.08100 -3.92600 -10.27100 + O 19.15700 -5.81300 -11.92800 + H 19.23200 -6.04200 -11.00200 + H 19.86900 -6.29200 -12.35200 + O 21.21700 -7.44400 -12.90100 + H 20.66600 -8.16600 -12.59800 + H 21.96000 -7.44000 -12.29600 + O 18.26700 -21.31200 -16.65700 + H 17.46300 -20.91100 -16.98700 + H 18.08900 -22.25300 -16.67300 + O 16.66300 -21.45100 -19.64600 + H 17.49000 -21.04600 -19.90700 + H 16.40000 -20.97400 -18.85900 + O 18.91400 -19.84100 -20.01900 + H 19.20200 -19.67100 -20.91600 + H 19.71600 -20.06400 -19.54700 + O 18.61500 -19.83800 -5.66400 + H 17.89500 -20.26900 -5.20400 + H 19.40300 -20.12800 -5.20400 + O 20.60500 -20.65300 -3.92900 + H 21.55600 -20.62800 -4.03200 + H 20.41100 -21.56600 -3.71400 + O -9.86500 8.09400 -16.65100 + H -10.05400 9.00600 -16.43100 + H -10.65000 7.78800 -17.10500 + O -10.23700 6.46600 -11.38900 + H -9.94700 5.95800 -10.63100 + H -10.87300 5.90100 -11.82800 + O -18.12200 6.17800 -10.23500 + H -17.96600 5.41900 -9.67400 + H -17.28000 6.33800 -10.66300 + O -6.09900 9.43400 -13.02800 + H -6.67700 8.69500 -12.83700 + H -6.36900 10.11400 -12.41100 + O -16.20400 10.33900 -13.14400 + H -16.67000 9.50500 -13.19900 + H -15.36600 10.11800 -12.73700 + O 21.62100 3.97400 -16.11700 + H 20.98700 4.31300 -16.74900 + H 22.40500 3.79500 -16.63600 + O -19.79100 6.62800 -13.32800 + H -19.40200 5.75300 -13.30100 + H -19.04600 7.21800 -13.44300 + O -8.67300 14.94900 -13.56800 + H -8.18700 14.52100 -14.27300 + H -9.28400 14.28100 -13.25800 + O -11.58100 8.93500 -11.07900 + H -11.14700 8.13200 -11.36800 + H -11.06300 9.23300 -10.33100 + O -3.16900 -0.75300 -10.44700 + H -3.28100 0.09500 -10.01800 + H -2.49500 -0.60000 -11.10900 + O -9.69100 2.75100 -11.82700 + H -9.86800 1.86600 -11.50800 + H -9.21700 2.61900 -12.64900 + O -15.37100 1.43400 -13.83800 + H -15.22200 2.29100 -14.23900 + H -15.90400 0.96000 -14.47500 + O -7.09800 -0.24700 -14.34000 + H -7.25200 -0.14800 -13.40000 + H -6.35000 -0.84100 -14.39900 + O -18.77900 4.06100 -13.29800 + H -18.15200 3.55900 -12.77800 + H -18.57500 3.83300 -14.20500 + O -14.05700 5.54800 -13.56300 + H -13.22900 5.19300 -13.24200 + H -14.43800 4.83900 -14.08100 + O -3.67500 1.51600 -8.97100 + H -4.13600 2.34100 -9.12200 + H -4.27800 0.99800 -8.43900 + O -16.82100 0.47700 -9.67500 + H -17.18300 1.17500 -10.22100 + H -17.55600 0.19000 -9.13400 + O -10.82300 10.66000 -16.46000 + H -11.19300 11.03100 -17.26100 + H -10.70900 11.41100 -15.87800 + O -14.16500 8.32600 -10.15700 + H -14.64900 9.08700 -9.83700 + H -13.35800 8.69000 -10.52000 + O -7.24100 0.19600 -11.62400 + H -8.13500 0.29000 -11.29400 + H -6.88000 1.08200 -11.59300 + O -17.07600 2.30200 -11.88800 + H -16.53400 2.87300 -11.34400 + H -16.46400 1.91700 -12.51500 + O -15.75800 10.24700 -8.95900 + H -15.72000 11.19500 -8.83000 + H -16.68200 10.02500 -8.84100 + O -19.93000 9.80900 -17.42100 + H -20.34700 10.41100 -18.03700 + H -19.81800 10.32300 -16.62100 + O -6.76700 11.47700 -11.27400 + H -7.63400 11.58400 -10.88300 + H -6.55400 12.34200 -11.62500 + O -5.84000 -1.59600 -10.03000 + H -6.35400 -1.05500 -10.62800 + H -4.93100 -1.44000 -10.28700 + O -13.64400 -2.93100 -12.68000 + H -13.88900 -3.80900 -12.38600 + H -14.25300 -2.34800 -12.22600 + O -7.60500 13.46500 -15.67300 + H -7.15200 12.64400 -15.47700 + H -6.93600 14.01800 -16.07600 + O -6.31100 11.09500 -15.32800 + H -6.10600 10.56400 -16.09700 + H -6.18700 10.50200 -14.58700 + O -5.41500 9.50200 -7.94300 + H -5.54600 9.02000 -8.76000 + H -5.33300 10.41600 -8.21400 + O -11.57000 4.88300 -17.64400 + H -11.87800 5.78900 -17.62200 + H -10.72400 4.92800 -18.09000 + O -11.38600 4.86900 -21.20300 + H -10.57800 5.35600 -21.04100 + H -11.24600 4.44600 -22.05000 + O -3.92300 5.94600 -9.18300 + H -4.53000 6.67200 -9.32400 + H -3.47200 5.84400 -10.02100 + O -7.36200 7.24800 -17.34400 + H -8.17300 7.56600 -16.94700 + H -6.94700 6.72600 -16.65800 + O -7.65900 4.07000 -7.57800 + H -7.67100 4.82500 -6.99000 + H -8.29700 4.28700 -8.25900 + O -15.32700 7.33100 -15.35800 + H -15.66300 6.81100 -16.08800 + H -14.76700 6.72700 -14.87000 + O -13.45400 10.50900 -12.46600 + H -13.27600 10.65100 -13.39600 + H -12.69700 10.01600 -12.14900 + O -15.59500 6.41600 -11.46900 + H -15.02100 6.16800 -12.19300 + H -15.10500 7.08400 -10.99000 + O -9.03000 11.72700 -9.75200 + H -9.37300 12.43800 -9.21000 + H -9.48400 10.94500 -9.43800 + O -14.77400 10.06400 -6.31600 + H -15.51400 10.54900 -5.94900 + H -15.02400 9.89600 -7.22400 + O -9.00300 5.44500 -18.75000 + H -8.46800 6.10900 -18.31300 + H -8.99100 5.69700 -19.67300 + O -17.53900 5.87200 -6.48500 + H -17.39200 6.81500 -6.42200 + H -18.38000 5.73100 -6.05000 + O -10.36200 13.14400 -15.28200 + H -9.42700 13.22200 -15.47400 + H -10.77300 13.82700 -15.81100 + O -16.91200 11.78100 -5.66300 + H -16.66600 12.22800 -6.47300 + H -17.86700 11.83100 -5.64100 + O -16.37200 6.15700 -17.61400 + H -17.18000 6.62400 -17.82700 + H -16.48000 5.29500 -18.01500 + O -4.90200 13.73800 -20.37100 + H -4.43700 14.09700 -19.61600 + H -4.52800 14.19500 -21.12400 + O -20.73500 8.84700 -8.68000 + H -21.55500 9.31700 -8.83100 + H -20.70200 8.19100 -9.37700 + O -21.22800 11.89700 -10.62000 + H -21.81700 11.26400 -10.20900 + H -20.54500 12.04900 -9.96700 + O -3.49000 12.11100 -5.50500 + H -3.58900 11.23900 -5.12300 + H -4.28300 12.57800 -5.24400 + O -12.31100 11.30200 -6.31600 + H -12.42300 12.14500 -5.87800 + H -13.17300 10.88900 -6.26700 + O -11.34900 15.25700 -16.83400 + H -11.41900 15.04900 -17.76600 + H -10.78700 16.03200 -16.80400 + O -17.80100 13.58800 -14.87800 + H -17.60300 13.32200 -13.98000 + H -18.43600 12.94100 -15.18500 + O -8.70100 15.02100 -20.14700 + H -8.23300 14.63400 -20.88700 + H -8.73200 14.32400 -19.49200 + O -8.55000 13.06200 -18.31000 + H -8.33500 13.08200 -17.37700 + H -7.91900 12.44900 -18.68800 + O -5.58900 2.62900 -6.22200 + H -6.28900 3.09000 -6.68400 + H -5.57800 1.75500 -6.61100 + O -18.34500 10.12400 -7.97300 + H -18.08500 9.65700 -7.17800 + H -19.12300 9.65800 -8.28100 + O -15.72100 -1.57400 -11.24500 + H -16.48400 -1.70000 -11.80900 + H -15.87800 -0.73300 -10.81400 + O -11.39900 -2.09600 -6.94100 + H -10.97400 -2.65100 -6.28600 + H -12.32500 -2.10600 -6.69800 + O -8.26700 1.32700 -7.26800 + H -8.63800 1.24100 -6.39000 + H -8.18000 2.27100 -7.40100 + O -8.88200 6.20700 -21.40900 + H -9.13500 6.52900 -22.27400 + H -7.98400 6.51500 -21.29200 + O -11.55700 -0.06300 -19.62600 + H -12.30000 0.44100 -19.95600 + H -11.88800 -0.48900 -18.83500 + O -15.49000 3.88700 -10.29100 + H -16.10800 3.91400 -9.56200 + H -15.46700 4.78600 -10.62000 + O -15.36400 3.79600 -15.20800 + H -16.27500 3.80600 -15.50100 + H -14.85800 3.62700 -16.00200 + O -10.44900 2.59300 -16.19600 + H -11.03700 3.31700 -16.41100 + H -9.74000 2.66200 -16.83500 + O -11.10300 3.97200 -5.73500 + H -11.53100 3.25600 -5.26700 + H -10.23800 4.04100 -5.33000 + O -8.48400 2.29200 -14.24000 + H -9.16000 2.35200 -14.91500 + H -8.03000 1.47000 -14.42900 + O -17.58800 4.04400 -8.57300 + H -17.61400 4.56900 -7.77300 + H -18.23100 3.35000 -8.42600 + O -13.98200 1.17000 -20.52000 + H -14.27100 1.20500 -19.60800 + H -14.75700 0.88900 -21.00600 + O -21.07900 -0.41300 -14.27300 + H -20.56400 -0.01300 -13.57200 + H -21.63700 0.29500 -14.59400 + O -20.10000 3.34200 -17.84000 + H -19.44300 2.99700 -18.44400 + H -20.71300 3.81900 -18.40000 + O -9.55400 4.36200 -9.59200 + H -9.45900 3.84100 -10.38900 + H -10.32000 3.99100 -9.15400 + O -12.77800 0.97800 -12.89800 + H -13.69000 1.01900 -13.18700 + H -12.78900 1.35100 -12.01700 + O -18.06100 3.22000 -15.83500 + H -18.77800 3.33500 -16.45800 + H -17.80500 2.30300 -15.92900 + O -13.84900 3.22600 -17.43500 + H -13.06200 3.74900 -17.58800 + H -14.48000 3.54900 -18.07900 + O -9.92500 0.20900 -10.84300 + H -10.25100 0.24200 -9.94400 + H -10.32700 -0.57500 -11.21600 + O -15.97300 3.81900 -19.05800 + H -16.77400 3.39300 -19.36300 + H -15.47400 3.99300 -19.85600 + O -5.65200 16.09300 -4.46900 + H -5.39900 16.46600 -5.31300 + H -4.91000 16.27700 -3.89300 + O -17.79500 8.56700 -5.65100 + H -17.44900 8.44900 -4.76600 + H -18.74100 8.65000 -5.52900 + O -11.93500 4.30600 -12.23000 + H -11.10900 3.82800 -12.30600 + H -12.49000 3.74200 -11.69200 + O -16.98300 0.65700 -15.95800 + H -17.52200 -0.12100 -15.82000 + H -16.44400 0.44300 -16.72000 + O -13.17900 2.44400 -10.47200 + H -13.99100 2.93200 -10.34200 + H -12.62700 2.69500 -9.73000 + O -12.22300 7.66200 -18.16300 + H -12.84700 7.37600 -18.83000 + H -12.70000 8.31800 -17.65400 + O -5.60900 7.90000 -10.17900 + H -4.95200 8.28600 -10.75800 + H -6.31600 7.63100 -10.76600 + O -8.77800 0.39100 -18.80200 + H -9.65500 0.23000 -19.14900 + H -8.77000 1.32500 -18.59300 + O -11.69400 8.41800 -6.60800 + H -11.19200 7.99200 -5.91400 + H -11.90400 9.28300 -6.25600 + O -8.32900 2.93900 -17.87500 + H -7.40700 2.98800 -17.62000 + H -8.52000 3.79800 -18.25000 + O -17.20600 7.75800 -13.38300 + H -16.65800 7.65000 -14.16100 + H -16.77300 7.23400 -12.71000 + O -10.50300 9.41600 -19.77600 + H -10.85800 10.18400 -19.32700 + H -10.83100 8.67100 -19.27300 + O -20.00400 5.86900 -5.26100 + H -19.95900 6.04900 -4.32200 + H -20.71700 5.23600 -5.34900 + O -13.10000 5.85000 -4.98400 + H -13.79500 5.48700 -5.53200 + H -12.29700 5.44600 -5.31400 + O -9.51300 11.01700 -6.04000 + H -10.42700 11.07600 -6.31800 + H -9.54100 11.15400 -5.09300 + O -4.24800 3.77600 -19.49700 + H -4.70800 3.41000 -20.25300 + H -4.34100 4.72300 -19.59500 + O -3.04400 8.30600 -7.19800 + H -3.82600 8.81600 -7.40800 + H -2.57300 8.23500 -8.02900 + O -6.49400 11.56800 -19.38100 + H -6.09900 10.98500 -18.73200 + H -5.80200 12.19400 -19.59400 + O -7.51700 7.10200 -12.10800 + H -8.45500 7.06900 -11.91900 + H -7.33500 6.28000 -12.56500 + O -14.41100 11.22500 -15.12400 + H -14.63900 12.07100 -15.50900 + H -15.15900 11.00500 -14.56900 + O -19.26800 5.46200 -20.01900 + H -19.02200 6.25000 -19.53400 + H -20.21400 5.38800 -19.89400 + O -14.17400 9.25900 -17.03900 + H -14.07200 10.02800 -16.47800 + H -14.65100 8.62900 -16.49800 + O -13.17200 12.68800 -8.68900 + H -12.61800 13.44000 -8.89700 + H -12.66300 12.18200 -8.05600 + O -15.36100 13.59400 -16.18000 + H -16.24100 13.68900 -15.81500 + H -14.92400 14.41900 -15.97100 + O -10.13700 9.34600 -8.72800 + H -10.78500 9.00700 -8.11000 + H -9.29600 9.05100 -8.37800 + O -7.93600 9.79000 -20.91600 + H -7.51000 10.40300 -20.31700 + H -8.78700 9.61400 -20.51500 + O -7.63000 12.80500 -6.93800 + H -8.20900 12.11900 -6.60400 + H -8.20900 13.39500 -7.41900 + O -19.46300 2.07300 -8.23800 + H -20.39500 2.29300 -8.26600 + H -19.44400 1.12200 -8.13600 + O -5.80400 9.55300 -17.61500 + H -6.39500 8.80000 -17.59600 + H -4.94100 9.17500 -17.78500 + O -19.42500 12.67500 -8.61700 + H -18.85800 11.95800 -8.33200 + H -19.08000 13.44700 -8.16800 + O -14.40000 9.88100 21.57700 + H -13.46800 10.09300 21.53000 + H -14.64900 10.09200 22.47700 + O -14.97800 10.25200 -19.62100 + H -14.68800 9.74800 -18.86100 + H -14.86000 11.16600 -19.36200 + O -18.53200 7.68300 -18.51300 + H -19.09400 8.23900 -17.97300 + H -18.18600 8.27100 -19.18400 + O -10.67200 0.28700 -8.17300 + H -9.78900 0.46800 -7.85400 + H -10.93400 -0.51300 -7.71700 + O -6.75300 -0.39300 -17.06800 + H -7.47500 -0.07300 -17.60900 + H -7.05700 -0.27400 -16.16800 + O -11.88500 11.64300 -18.84100 + H -12.77600 11.93600 -18.64800 + H -11.62600 12.15200 -19.60900 + O -11.28100 0.18800 -15.08900 + H -11.01100 1.03600 -15.44100 + H -11.73100 0.40000 -14.27100 + O -11.55500 2.93400 -8.30600 + H -11.51600 3.34000 -7.44000 + H -11.31300 2.02100 -8.15400 + O -10.47500 -2.46500 -15.34100 + H -10.60700 -1.52800 -15.19200 + H -9.95400 -2.50800 -16.14300 + O -14.42600 12.77000 -18.64300 + H -14.46200 13.59700 -19.12400 + H -14.81100 12.97100 -17.79000 + O -14.77500 0.63900 -17.83200 + H -14.50600 1.51300 -17.55200 + H -14.06000 0.06700 -17.55400 + O -5.80800 0.19300 -7.78400 + H -6.72600 0.43200 -7.65400 + H -5.83900 -0.58500 -8.34000 + O -7.11400 4.51300 -13.27500 + H -7.71200 3.89600 -13.69600 + H -6.63200 3.98400 -12.63900 + O -17.44200 9.32100 -20.44500 + H -16.64100 9.71300 -20.09800 + H -17.13600 8.67700 -21.08400 + O -18.37300 2.89100 -20.01100 + H -18.66300 3.78700 -20.18400 + H -18.68100 2.38900 -20.76500 + O -5.81800 14.02900 -11.86700 + H -6.02600 14.90900 -12.18100 + H -4.88900 13.91500 -12.06900 + O -18.11600 -1.68000 -12.59400 + H -18.13500 -1.65400 -13.55000 + H -18.72100 -2.38400 -12.36000 + O -19.97500 12.29600 -15.98500 + H -20.72400 12.75400 -15.60200 + H -20.09800 12.38200 -16.93000 + O -20.41100 14.22300 -3.68400 + H -19.76300 14.14000 -2.98400 + H -20.25800 13.46200 -4.24400 + O -16.24500 12.30100 -3.04800 + H -15.40300 11.86800 -2.90900 + H -16.44900 12.14100 -3.96900 + O -11.60700 12.92400 -21.31300 + H -11.73900 12.06700 -21.71700 + H -11.45900 13.51600 -22.05100 + O -9.62700 11.44500 -3.34000 + H -9.50100 10.56900 -2.97500 + H -8.84100 11.92800 -3.08200 + O -7.20600 13.47200 -21.95100 + H -7.14800 12.60700 -22.35600 + H -6.43600 13.52400 -21.38500 + O -20.54000 8.46700 -5.97400 + H -20.40800 7.52100 -5.92100 + H -20.67700 8.63900 -6.90500 + O -15.82400 7.56700 -21.91800 + H -15.39300 7.26800 -21.11700 + H -15.31200 8.32600 -22.19600 + O -14.36300 8.25200 -4.23700 + H -14.42300 8.74800 -5.05400 + H -13.82800 7.49000 -4.45900 + O -11.68800 10.26000 21.78800 + H -11.29900 9.88300 22.57700 + H -11.03500 10.11200 21.10400 + O -7.94300 8.86700 -6.98800 + H -7.07100 9.19200 -7.21400 + H -8.37800 9.61400 -6.57700 + O -19.64900 7.28400 -2.78400 + H -18.69200 7.29300 -2.78800 + H -19.88300 7.04400 -1.88800 + O -14.50100 6.87000 -19.55400 + H -15.10600 6.72600 -18.82600 + H -14.38600 6.00400 -19.94500 + O -14.99500 4.76900 -6.69300 + H -15.88400 5.11200 -6.59900 + H -14.55500 5.39700 -7.26600 + O -14.08500 4.55300 -20.91600 + H -14.24400 4.15200 -21.77000 + H -13.13800 4.68700 -20.88600 + O -10.02900 7.22600 -4.63400 + H -9.25900 6.86600 -5.07300 + H -10.27400 6.55800 -3.99300 + O -6.43600 7.50100 -21.43000 + H -5.87700 7.66000 -22.19100 + H -6.85700 8.34400 -21.26100 + O -14.68400 2.15200 -5.86800 + H -14.61600 1.63400 -6.67000 + H -14.76700 3.05500 -6.17300 + O -14.45500 0.73800 -8.28900 + H -13.83900 1.03600 -8.95800 + H -15.29500 0.68600 -8.74300 + O -13.66000 6.80900 -7.90100 + H -12.96800 7.30200 -7.46100 + H -13.82400 7.29300 -8.71000 + O -8.74200 3.61400 -4.22300 + H -8.96200 2.70200 -4.41000 + H -7.85100 3.57900 -3.87600 + O -7.30600 6.37100 -6.00500 + H -7.54600 7.21000 -6.39900 + H -6.34900 6.36100 -6.02600 + O -16.29000 2.07600 -3.58900 + H -17.20500 1.95600 -3.84200 + H -15.81100 2.09000 -4.41800 + O -14.27100 -1.75400 -7.01800 + H -14.23600 -0.86000 -7.35900 + H -14.93400 -2.18900 -7.55400 + O -12.22300 1.75400 -4.55700 + H -12.32000 1.46100 -3.65100 + H -13.11900 1.79600 -4.89300 + O -9.56300 1.09900 -4.89900 + H -10.51400 1.17000 -4.81900 + H -9.31300 0.45600 -4.23600 + O -8.73800 -0.54000 -2.88800 + H -8.71000 -1.49200 -2.79400 + H -9.06400 -0.22500 -2.04500 + O -19.02800 1.17500 -3.85700 + H -19.10600 0.46400 -4.49200 + H -18.89700 0.73400 -3.01700 + O -14.59400 -2.77600 -4.47500 + H -14.10600 -3.58800 -4.61000 + H -14.53500 -2.31900 -5.31400 + O -13.51700 -1.94000 -2.05400 + H -13.84000 -2.29000 -2.88400 + H -14.11800 -1.22400 -1.84600 + O -14.18600 15.34600 -19.74600 + H -13.23000 15.31000 -19.79800 + H -14.36700 16.10200 -19.18800 + O -19.22400 11.41600 -20.83300 + H -18.67900 10.66200 -20.60800 + H -19.38300 11.32500 -21.77200 + O 21.91200 13.95100 -15.11700 + H 21.02800 13.74700 -15.42200 + H 21.85200 13.90600 -14.16200 + O 21.71000 13.70400 -12.40000 + H 22.20800 13.11800 -11.83100 + H 21.66800 14.52900 -11.91800 + O -3.04000 14.04800 -12.20800 + H -2.97200 14.53900 -11.38900 + H -2.16500 14.09900 -12.59200 + O -4.01700 14.08500 -7.66200 + H -4.53900 13.44900 -8.15000 + H -3.59800 13.57200 -6.97100 + O -4.53800 10.27500 -3.37600 + H -5.37400 10.13200 -2.93300 + H -4.02500 10.79600 -2.75800 + O -3.45000 8.24700 -18.11800 + H -2.58800 8.63200 -18.27900 + H -3.31600 7.66200 -17.37200 + O -3.40600 6.37900 -16.13600 + H -4.29400 6.17400 -15.84400 + H -2.87600 6.33900 -15.34000 + O -3.63100 9.02900 -11.72200 + H -4.27000 9.16300 -12.42200 + H -3.21800 9.88600 -11.60700 + O -5.61700 12.12000 -8.77200 + H -6.38700 12.38900 -8.27100 + H -5.95500 11.93000 -9.64700 + O -4.38400 6.44100 -19.92100 + H -5.17000 6.76700 -20.35900 + H -4.16000 7.12500 -19.29000 + O -5.45700 2.93100 -21.83300 + H -5.93400 2.77200 -22.64800 + H -5.76900 2.25100 -21.23500 + O -6.00200 5.46300 -15.63900 + H -5.91800 4.63800 -16.11600 + H -6.35900 5.21300 -14.78700 + O -20.67900 6.93000 -10.69200 + H -20.64500 7.01600 -11.64400 + H -19.79400 6.66700 -10.44200 + O -4.47700 6.27700 -5.92900 + H -3.96100 5.47100 -5.91000 + H -3.96000 6.88000 -6.46200 + O -3.09300 3.69100 -5.88100 + H -3.98300 3.36100 -5.99900 + H -2.54200 2.90900 -5.90800 + O -6.27900 1.00500 -20.10600 + H -5.59700 0.36200 -19.91200 + H -7.07300 0.63700 -19.71700 + O -5.63300 3.13100 -17.16300 + H -5.15000 2.37900 -16.82100 + H -5.19100 3.34700 -17.98400 + O -5.85000 2.62800 -11.65800 + H -4.97000 2.65800 -12.03400 + H -5.74000 2.95700 -10.76600 + O -5.39600 3.66900 -9.18100 + H -6.18800 3.84800 -8.67400 + H -4.88000 4.47200 -9.11100 + O -20.78500 3.36000 -3.41500 + H -20.16200 2.66600 -3.63200 + H -21.25800 3.52200 -4.23100 + O -3.27800 -2.53800 -3.30000 + H -2.46400 -2.99300 -3.51800 + H -3.39200 -1.90000 -4.00400 + O -4.23600 0.81300 -16.40400 + H -4.07200 0.66200 -15.47300 + H -4.98300 0.25000 -16.61000 + O -19.37300 0.76500 -12.22000 + H -18.99600 -0.11500 -12.21100 + H -18.61800 1.35000 -12.15300 + O -20.92200 3.07900 -11.78000 + H -20.36200 3.62200 -12.33500 + H -20.47900 2.23200 -11.74800 + O -3.91000 -0.58700 -5.21500 + H -4.69900 -0.25500 -4.78700 + H -4.23700 -1.18000 -5.89200 + O -3.87300 -0.47300 -20.18300 + H -3.36500 -1.16800 -19.76600 + H -3.30900 0.29900 -20.13400 + O -5.79600 15.15100 -16.87100 + H -5.10600 14.82000 -17.44700 + H -5.37600 15.85300 -16.37300 + O -21.01000 -2.01000 -9.28400 + H -21.56100 -2.75800 -9.05800 + H -21.59900 -1.40400 -9.73300 + O -19.74400 11.83200 -5.04000 + H -20.65700 11.68600 -5.29000 + H -19.59800 11.23400 -4.30600 + O -15.81900 12.85400 -7.98900 + H -16.08600 13.72200 -8.28900 + H -14.87000 12.83800 -8.11400 + O -12.52200 13.81800 -5.13500 + H -12.34000 13.52900 -4.24000 + H -11.79800 14.40600 -5.34900 + O -10.35400 15.41500 -5.82100 + H -9.87000 14.99500 -6.53100 + H -9.68400 15.65000 -5.18000 + O -4.73400 -3.01100 -6.54400 + H -5.27800 -3.05600 -5.75800 + H -4.67300 -3.91700 -6.84600 + O -18.94400 15.32100 -7.60200 + H -19.57200 15.92600 -7.99700 + H -18.88600 15.59800 -6.68800 + O -16.46000 15.37600 -8.77800 + H -17.36200 15.41000 -8.46000 + H -16.50700 15.71500 -9.67100 + O -11.91900 15.01500 -9.53700 + H -12.49700 15.24500 -10.26500 + H -11.95500 15.77400 -8.95400 + O -9.55900 14.21100 -8.33500 + H -10.33300 14.49300 -8.82300 + H -8.82000 14.53000 -8.85300 + O -7.27600 14.84200 -9.61800 + H -6.78000 15.65300 -9.50500 + H -6.84200 14.39500 -10.34500 + O -18.92800 14.20500 -11.02100 + H -19.22400 13.67400 -10.28200 + H -18.28100 13.65900 -11.46600 + O -16.94900 12.80200 -12.30500 + H -16.86300 11.89400 -12.59400 + H -16.10900 13.00200 -11.89100 + O -14.36300 12.82700 -11.21800 + H -14.05600 12.06600 -11.71100 + H -14.00900 12.70100 -10.33800 + O -11.01600 -2.21200 -11.70200 + H -11.82300 -2.37600 -12.19100 + H -10.34500 -2.70500 -12.17300 + O -6.27500 -4.12700 -13.83000 + H -5.86300 -3.27400 -13.96900 + H -7.18600 -3.92200 -13.62000 + O -4.78000 -1.82300 -14.34100 + H -4.09400 -2.41100 -14.65600 + H -4.30800 -1.05800 -14.01100 + O -18.55300 -1.52000 -15.33300 + H -19.50300 -1.43900 -15.24600 + H -18.41300 -2.42700 -15.60400 + O -13.14500 -3.25800 -15.42400 + H -12.19700 -3.14900 -15.35300 + H -13.47100 -3.11300 -14.53600 + O -9.08500 -3.71700 -13.31600 + H -9.56300 -3.34100 -14.05500 + H -9.36000 -4.63400 -13.29400 + O -6.74100 -2.93100 -18.14500 + H -6.61600 -2.03900 -17.82100 + H -7.65800 -3.12900 -17.95600 + O -20.54800 12.51400 -18.66800 + H -20.18400 12.14200 -19.47200 + H -20.54800 13.45900 -18.81900 + O -15.68700 -1.37800 -19.58800 + H -16.06400 -0.86800 -20.30600 + H -15.47600 -0.72800 -18.91800 + O -12.68600 -0.98200 -17.14600 + H -12.98900 -1.79800 -16.74800 + H -12.18400 -0.55000 -16.45500 + O -11.42700 15.07600 -19.63100 + H -10.53000 15.35200 -19.81900 + H -11.54800 14.28200 -20.15100 + O -18.89200 -1.29900 -5.32700 + H -19.44900 -2.06400 -5.18200 + H -18.12900 -1.44800 -4.76900 + O -17.06800 -2.07600 -3.35500 + H -16.19300 -2.28100 -3.68600 + H -17.40200 -2.91100 -3.02800 + O -18.02600 13.83600 -20.15200 + H -18.73700 14.29900 -19.70900 + H -18.40200 12.98700 -20.38500 + O -8.32200 -3.24400 -2.97400 + H -8.32700 -3.97400 -2.35600 + H -7.47600 -3.31000 -3.41700 + O -5.84400 13.37000 -4.89400 + H -6.42300 13.24100 -5.64600 + H -5.81300 14.31900 -4.77400 + O 9.66800 6.83900 -14.79400 + H 9.25800 7.61800 -15.16800 + H 8.93600 6.25300 -14.59900 + O 6.63700 5.80900 -11.35600 + H 7.42100 5.45900 -10.93300 + H 6.45700 5.19800 -12.07000 + O -0.07200 6.15300 -10.77900 + H 0.16500 5.45800 -10.16400 + H 0.70500 6.26200 -11.32800 + O 12.13700 9.88700 -13.08100 + H 11.97700 8.97600 -12.83300 + H 12.06000 10.37600 -12.26100 + O 2.25700 9.85500 -13.43300 + H 1.62500 9.21100 -13.75200 + H 2.71400 9.41100 -12.71900 + O -2.23800 4.35300 -17.60300 + H -2.58200 5.10900 -17.12700 + H -2.90100 4.16600 -18.26800 + O -1.58700 6.06900 -14.09500 + H -1.20000 5.20100 -13.97900 + H -0.83700 6.65100 -14.21400 + O 8.54600 12.76400 -12.38500 + H 8.20000 12.10400 -12.98500 + H 8.69100 12.29100 -11.56600 + O 6.37700 8.57700 -11.12900 + H 6.56600 7.65000 -11.27600 + H 6.90600 8.81200 -10.36700 + O 9.57900 2.91600 -12.38800 + H 9.51800 1.98000 -12.19700 + H 9.11800 3.02100 -13.22000 + O 2.79500 0.72400 -14.71300 + H 2.76000 1.58100 -15.13800 + H 2.09500 0.22100 -15.12800 + O 10.88300 1.97100 -15.14200 + H 11.27200 1.66100 -14.32500 + H 11.62800 2.26400 -15.66700 + O -0.73400 3.37400 -13.72100 + H -0.08200 2.84100 -13.26500 + H -0.58800 3.18800 -14.64900 + O 3.74600 4.91700 -14.36500 + H 4.58900 4.56900 -14.07500 + H 3.34800 4.20000 -14.86000 + O 16.12800 0.34300 -9.25100 + H 16.29600 1.14300 -9.75000 + H 15.33900 0.53800 -8.74500 + O 1.22900 0.51500 -9.94800 + H 1.07400 0.94400 -10.78900 + H 0.51400 0.81800 -9.38800 + O 8.20200 8.88400 -15.99200 + H 8.19200 9.23700 -16.88200 + H 8.01700 9.63700 -15.43200 + O 3.60300 8.43700 -11.24900 + H 3.27800 9.04200 -10.58200 + H 4.53800 8.62900 -11.31100 + O 12.46700 1.11400 -13.05700 + H 13.05200 0.39100 -13.28300 + H 12.93800 1.60300 -12.38300 + O 1.18600 1.62600 -12.54600 + H 1.71300 2.31300 -12.13800 + H 1.77200 1.22300 -13.18700 + O 2.69300 10.16400 -9.30200 + H 2.85200 11.09400 -9.13800 + H 1.74100 10.07300 -9.26000 + O -1.02500 9.41400 -18.74000 + H -1.21700 9.91800 -19.53100 + H -0.81500 10.07500 -18.08000 + O 11.79100 11.59200 -10.79000 + H 10.85700 11.54200 -10.58800 + H 11.93900 12.51000 -11.01700 + O 11.52300 -1.59900 -12.63600 + H 11.35000 -0.66400 -12.52800 + H 12.41000 -1.64000 -12.99500 + O 4.87000 -3.16600 -11.57100 + H 4.56400 -4.06300 -11.70600 + H 4.07600 -2.67100 -11.36600 + O 10.16200 12.70100 -15.63300 + H 10.84200 12.03000 -15.69700 + H 10.56400 13.40100 -15.11900 + O 12.14300 10.77800 -15.67000 + H 12.07600 9.96500 -16.17200 + H 12.18300 10.49300 -14.75700 + O 13.07400 8.83200 -7.81000 + H 12.96400 8.40500 -8.65900 + H 13.40100 9.70600 -8.02200 + O 7.22900 6.48100 -17.25800 + H 6.75300 6.88400 -17.98400 + H 7.51600 7.21900 -16.72000 + O 8.47700 3.81800 -20.25700 + H 9.30300 4.16900 -19.92400 + H 8.67200 3.55500 -21.15600 + O 14.26000 5.62300 -8.85000 + H 13.68000 6.21900 -9.32300 + H 15.09000 5.67000 -9.32400 + O 11.46700 5.89400 -16.71000 + H 10.81300 6.15000 -16.06000 + H 12.19800 5.55300 -16.19400 + O 10.32200 3.86900 -7.94100 + H 10.40700 4.66700 -7.42000 + H 9.81000 4.13400 -8.70500 + O 2.58200 6.54600 -16.41000 + H 2.44300 6.00100 -17.18500 + H 3.11000 6.00100 -15.82500 + O 5.65000 10.69000 -12.80300 + H 6.00300 10.53200 -13.67900 + H 6.00200 9.97700 -12.27000 + O 2.35700 6.19900 -12.35300 + H 2.92800 5.82100 -13.02100 + H 2.85700 6.93300 -11.99600 + O 9.09600 11.44700 -9.96100 + H 8.93700 12.12200 -9.30100 + H 8.64200 10.67300 -9.62600 + O 3.99700 9.87300 -6.77800 + H 3.35400 10.46900 -6.39300 + H 3.69300 9.73900 -7.67500 + O 10.88400 4.74000 -19.21900 + H 11.05700 5.10400 -18.35100 + H 11.11100 5.44500 -19.82500 + O 0.94600 6.21100 -7.28900 + H 1.01500 7.12200 -7.00300 + H 0.00800 6.02300 -7.26700 + O 7.79300 11.44000 -14.86500 + H 8.63000 11.81100 -15.14300 + H 7.15700 12.13700 -15.02200 + O 1.81100 11.61300 -6.09100 + H 2.16700 12.11000 -6.82800 + H 0.95500 12.00800 -5.92600 + O 2.14600 5.27000 -18.89700 + H 1.37800 5.80600 -19.09400 + H 2.05100 4.50000 -19.45800 + O 14.40300 10.91800 -19.66400 + H 14.37300 11.82400 -19.97200 + H 15.27600 10.61000 -19.90900 + O -1.77100 8.21800 -9.75100 + H -2.36400 8.41000 -10.47700 + H -1.18700 7.54000 -10.09100 + O -2.50600 11.46800 -11.33500 + H -1.77000 11.78700 -10.81300 + H -2.89300 12.25800 -11.71200 + O 15.92600 12.32200 -5.10700 + H 15.89000 11.37700 -5.25700 + H 15.02700 12.56300 -4.88100 + O 6.35200 11.51300 -7.01500 + H 6.01300 12.23600 -6.48700 + H 5.72400 10.80400 -6.88200 + O 6.91300 15.27800 -17.52500 + H 6.65900 15.23500 -18.44700 + H 6.84100 16.20500 -17.30000 + O 1.46200 14.18500 -15.46600 + H 1.67600 13.69200 -14.67400 + H 0.61300 13.83900 -15.74100 + O 9.24900 15.01500 -20.38300 + H 9.83700 14.48900 -20.92400 + H 9.51900 14.83100 -19.48400 + O 9.57800 14.56600 -17.63000 + H 8.64000 14.72200 -17.52600 + H 9.76900 13.84200 -17.03300 + O 13.16100 2.21800 -4.99200 + H 13.20800 1.60000 -5.72200 + H 13.10100 1.66500 -4.21300 + O -0.03000 10.16400 -8.83900 + H 0.15500 9.75300 -7.99500 + H -0.69300 9.60300 -9.24000 + O 2.38200 -1.87000 -10.98100 + H 1.72100 -2.06500 -11.64500 + H 2.09500 -1.04400 -10.59200 + O 7.09700 -1.94900 -8.04400 + H 7.48800 -2.37100 -7.27900 + H 6.16700 -1.87300 -7.82700 + O 10.43900 1.13300 -7.38900 + H 10.00400 1.13100 -6.53600 + H 10.45200 2.05400 -7.65000 + O 11.34400 6.99800 -20.79700 + H 10.86200 7.67200 -20.31800 + H 12.23300 7.34500 -20.87000 + O 6.20800 -0.58100 -21.41800 + H 5.30800 -0.61700 -21.74400 + H 6.13000 -0.75100 -20.48000 + O 2.48900 3.61600 -11.13400 + H 1.93300 3.69200 -10.35900 + H 2.55800 4.51000 -11.46900 + O 2.39700 3.12000 -16.00900 + H 1.44400 3.16400 -16.08700 + H 2.69600 2.84100 -16.87500 + O 7.51300 3.73700 -17.55200 + H 7.38000 4.67300 -17.40100 + H 7.82800 3.68200 -18.45500 + O 7.39800 3.63000 -6.62700 + H 7.12600 2.95500 -6.00500 + H 8.35300 3.65100 -6.55500 + O 8.35000 3.11700 -14.92400 + H 8.02500 3.34500 -15.79500 + H 9.20800 2.72400 -15.08400 + O 0.64700 3.99700 -9.14400 + H 0.68300 4.57300 -8.38100 + H 0.07400 3.27800 -8.87600 + O 3.46300 -0.25900 -21.90900 + H 3.11200 -0.27900 -21.01900 + H 3.30700 0.63600 -22.21000 + O -3.44300 0.51700 -13.72500 + H -2.61900 0.22800 -13.33200 + H -3.59300 1.38700 -13.35600 + O -2.24900 1.76700 -19.80900 + H -1.54900 2.12800 -20.35300 + H -2.85500 2.49700 -19.68300 + O 8.70700 4.29700 -10.19000 + H 9.09700 3.86100 -10.94800 + H 8.00800 3.70800 -9.90700 + O 5.49200 -0.08500 -15.30600 + H 4.60900 0.17800 -15.04800 + H 5.71300 -0.80000 -14.70900 + O -0.32700 2.87500 -16.37500 + H -0.90100 3.48800 -16.83400 + H -0.60500 2.01200 -16.68100 + O 3.17400 2.07300 -18.43200 + H 4.12800 2.06700 -18.35300 + H 3.00400 2.50600 -19.26900 + O 9.06500 0.27100 -11.77700 + H 8.77000 0.27800 -10.86700 + H 8.85200 -0.60800 -12.09100 + O 2.28300 3.15600 -20.83800 + H 1.34800 3.09600 -21.03200 + H 2.71200 3.03800 -21.68500 + O 13.00100 15.57300 -4.91800 + H 13.23300 15.78100 -5.82300 + H 13.70200 15.96200 -4.39500 + O 0.73900 9.00700 -6.44500 + H 1.21800 9.79200 -6.17900 + H -0.00700 8.96800 -5.84600 + O 6.09200 3.77600 -13.25800 + H 6.80600 3.43600 -13.79800 + H 5.81100 3.02600 -12.73300 + O 0.58700 -0.38800 -15.90800 + H 0.40300 -1.31100 -15.73200 + H -0.14300 -0.09900 -16.45600 + O 4.89900 2.20700 -11.14900 + H 4.13300 2.77600 -11.07400 + H 5.48800 2.50700 -10.45600 + O 5.80200 7.96900 -19.23300 + H 5.30900 7.39200 -19.81600 + H 5.14300 8.33300 -18.64200 + O 12.84300 7.55500 -10.24200 + H 13.65300 7.97600 -10.52700 + H 12.33400 7.44500 -11.04500 + O 12.49100 -0.37700 -19.67800 + H 12.33300 -0.94000 -18.92000 + H 12.05600 0.44700 -19.45900 + O 7.14200 8.54300 -6.15400 + H 7.75000 7.96900 -5.68800 + H 6.36300 8.56800 -5.59900 + O 11.34600 2.01600 -18.68800 + H 11.98700 2.20500 -18.00300 + H 11.09400 2.87500 -19.02500 + O 0.88200 7.43800 -14.35900 + H 1.35900 7.24400 -15.16600 + H 1.33700 6.93400 -13.68400 + O 7.87500 9.78600 -18.58800 + H 7.56800 10.60800 -18.97000 + H 7.20800 9.14400 -18.83100 + O -1.66700 5.38500 -7.56600 + H -2.07200 4.77500 -6.95000 + H -2.38700 5.67400 -8.12600 + O 5.46000 5.50400 -5.75100 + H 4.71200 5.26300 -6.29600 + H 6.20300 5.05000 -6.14900 + O 9.08700 11.07200 -6.46400 + H 8.18300 11.17800 -6.76100 + H 9.01800 10.97100 -5.51500 + O 14.59500 3.71200 -18.64800 + H 14.47600 3.37800 -19.53800 + H 14.45300 4.65500 -18.72500 + O 15.64300 9.64700 -5.46700 + H 14.77000 9.43300 -5.13800 + H 15.70000 9.19400 -6.30900 + O 14.19900 12.00700 -17.05600 + H 13.53000 11.57800 -16.52200 + H 14.16400 11.55100 -17.89700 + O 11.57700 7.09000 -12.69900 + H 10.92500 7.13900 -13.39800 + H 11.79900 6.16100 -12.64300 + O 4.05400 10.37900 -15.55400 + H 4.03300 11.31300 -15.76000 + H 3.50700 10.29300 -14.77300 + O -1.90800 5.51600 -20.74400 + H -1.18800 6.02200 -20.36700 + H -2.69800 5.91700 -20.38300 + O 3.70200 8.76600 -17.70600 + H 3.87100 9.37300 -16.98500 + H 3.22900 8.03900 -17.30100 + O 5.57900 12.81400 -9.32300 + H 6.13100 13.58300 -9.46400 + H 6.04400 12.30200 -8.66200 + O 3.97700 13.06400 -16.22400 + H 3.24200 13.62700 -15.98200 + H 4.72800 13.43700 -15.76200 + O 7.92200 9.08300 -8.83600 + H 7.43000 9.01400 -8.01800 + H 8.79800 8.76500 -8.61600 + O 10.49900 9.19300 -19.35200 + H 11.08300 9.12500 -18.59600 + H 9.64900 9.42400 -18.97900 + O 11.44700 12.41400 -6.88000 + H 10.59700 11.99100 -6.75900 + H 11.25200 13.20800 -7.37800 + O 14.46400 13.62900 -20.36300 + H 14.20900 13.76600 -21.27600 + H 13.99500 14.30900 -19.87900 + O -0.91700 1.72500 -8.47700 + H -1.80700 1.70900 -8.83000 + H -1.02500 1.53400 -7.54500 + O 12.27600 8.49600 -17.23900 + H 11.93300 7.60800 -17.13800 + H 13.21700 8.40800 -17.08900 + O -0.40800 12.72200 -9.93100 + H -0.18900 11.91100 -9.47300 + H -0.27400 13.41000 -9.27900 + O 4.21000 9.45600 21.06800 + H 5.11300 9.51200 21.38100 + H 3.67600 9.63800 21.84200 + O 3.34500 10.13900 -20.14000 + H 3.38800 9.58000 -19.36500 + H 3.56900 11.01300 -19.81900 + O 0.14400 7.03600 -19.73800 + H -0.15300 7.85600 -19.34400 + H 0.64500 7.30800 -20.50800 + O 8.32700 0.31800 -9.07100 + H 9.11200 0.43900 -8.53700 + H 7.91700 -0.47100 -8.71800 + O 9.84500 0.28300 -17.05400 + H 10.12200 0.84100 -17.78100 + H 10.08800 0.77500 -16.26900 + O 6.45400 11.73900 -20.09600 + H 5.68800 11.97000 -19.57100 + H 6.42800 12.34300 -20.83800 + O 5.78000 1.52700 -17.63900 + H 6.38700 2.26300 -17.56700 + H 5.76100 1.14500 -16.76100 + O 6.77300 2.63400 -9.17000 + H 6.83000 2.99100 -8.28300 + H 7.22400 1.79100 -9.11900 + O 7.63300 -1.34000 -16.83400 + H 7.07800 -0.93400 -16.16800 + H 8.38900 -0.75700 -16.90500 + O 4.07900 12.54600 -18.95200 + H 3.82000 13.39100 -19.32000 + H 4.02000 12.66900 -18.00500 + O 2.83400 -0.56300 -19.21200 + H 2.74600 0.28600 -18.77900 + H 3.65200 -0.92300 -18.87100 + O 13.28700 0.71500 -7.38800 + H 12.36600 0.46400 -7.45900 + H 13.33800 1.56600 -7.82500 + O 12.02300 4.31100 -12.64600 + H 11.19600 3.83800 -12.55700 + H 12.64800 3.80100 -12.13100 + O 0.95400 9.82800 -21.45400 + H 1.74400 10.06300 -20.96800 + H 1.15700 8.97900 -21.84700 + O -0.45100 3.25900 -21.38300 + H -0.87800 4.05900 -21.08000 + H -0.57500 3.26500 -22.33300 + O 12.62900 14.33800 -10.92400 + H 12.48400 15.07200 -11.52100 + H 13.57700 14.31900 -10.79400 + O 0.27200 -2.47000 -12.63600 + H 0.23400 -2.55100 -13.58900 + H 0.09300 -3.35400 -12.31300 + O -1.99400 12.37800 -18.43300 + H -2.59900 13.11700 -18.48900 + H -1.86800 12.10100 -19.34100 + O -1.10400 15.04600 -4.11600 + H -0.43200 14.86700 -3.45800 + H -1.07400 14.29000 -4.70100 + O 2.58400 12.31300 -3.43600 + H 3.27500 11.69400 -3.20000 + H 2.36700 12.09500 -4.34300 + O 5.82600 13.49100 21.74300 + H 4.91700 13.54200 21.44500 + H 6.33800 13.83600 21.01200 + O 9.09400 10.87900 -3.70600 + H 9.20600 10.03000 -3.27800 + H 9.84200 11.39800 -3.40800 + O 10.93100 13.40400 21.98100 + H 10.83500 12.46300 22.13000 + H 11.85700 13.52000 21.76600 + O -1.60700 8.97000 -4.93400 + H -1.30700 8.12700 -4.59600 + H -2.16300 8.74600 -5.68000 + O 1.73100 7.25300 21.89400 + H 2.60300 7.02500 22.21600 + H 1.59400 6.67700 21.14200 + O 4.56200 7.98100 -4.81200 + H 4.32500 8.48500 -5.59100 + H 4.88400 7.14700 -5.15500 + O 6.62200 9.40200 -21.50400 + H 6.60400 10.25800 -21.07500 + H 6.38500 8.78300 -20.81400 + O 10.31800 8.74800 -7.41100 + H 11.24700 8.95700 -7.50900 + H 9.93000 9.54400 -7.04600 + O -0.36700 6.76300 -3.98400 + H 0.57300 6.84200 -3.82300 + H -0.69400 6.24900 -3.24600 + O 4.01600 6.40500 -20.66500 + H 3.50400 5.94500 -19.99900 + H 4.66700 5.76400 -20.95100 + O 3.40900 4.86400 -7.49600 + H 2.56700 5.29700 -7.35200 + H 3.72400 5.22100 -8.32600 + O 6.13300 4.88200 -21.38900 + H 6.48900 5.12000 -22.24500 + H 6.88100 4.52500 -20.90900 + O 8.73000 6.81700 -4.60600 + H 9.38600 6.52400 -5.23800 + H 8.58800 6.05900 -4.03900 + O 13.71200 8.39600 -20.81500 + H 14.18700 8.46100 -21.64300 + H 13.79100 9.26600 -20.42400 + O 4.05100 2.25600 -6.70000 + H 3.97300 1.69200 -7.47000 + H 3.83800 3.13200 -7.02100 + O 3.80300 0.60000 -8.96100 + H 4.28400 0.89700 -9.73400 + H 2.88600 0.59200 -9.23600 + O 4.31200 6.03800 -9.81000 + H 5.14900 5.90800 -10.25500 + H 3.99900 6.88400 -10.13200 + O 10.77600 3.43600 -4.25800 + H 10.15500 2.75500 -4.51900 + H 11.61400 3.14900 -4.62100 + O 10.67900 6.18700 -6.50000 + H 10.47500 7.03400 -6.89900 + H 11.61200 6.23800 -6.29400 + O 2.62600 2.02100 -4.29800 + H 1.79300 1.57800 -4.46000 + H 2.98900 2.17400 -5.17000 + O 4.35700 -1.86500 -7.68500 + H 4.16000 -0.99700 -8.03600 + H 3.67500 -2.43100 -8.04600 + O 6.30100 1.52800 -5.16500 + H 5.90700 1.17700 -4.36600 + H 5.55700 1.73500 -5.72900 + O 9.09200 1.37500 -4.98900 + H 8.13600 1.31900 -4.97500 + H 9.38000 0.71900 -4.35500 + O 10.09900 -0.54300 -3.24100 + H 10.24000 -1.37500 -3.69300 + H 9.37700 -0.71600 -2.63700 + O 0.40900 0.25300 -4.53600 + H 0.39800 -0.56000 -5.04100 + H 0.56900 -0.02700 -3.63500 + O 4.70900 -3.26500 -5.32100 + H 5.18800 -4.02500 -5.65200 + H 4.54600 -2.72800 -6.09600 + O 5.78100 -1.99100 -3.12200 + H 5.47600 -2.50300 -3.87100 + H 5.35800 -1.13900 -3.22500 + O 3.72100 15.06000 -20.13800 + H 4.64100 15.31600 -20.07100 + H 3.26600 15.86600 -20.38100 + O -1.36900 11.12000 -20.89400 + H -0.51000 10.76900 -21.13000 + H -1.78800 11.31100 -21.73300 + O -1.11000 13.31700 -16.05200 + H -1.32400 12.90700 -16.89000 + H -1.65600 12.85600 -15.41400 + O 15.30400 14.49000 -10.26100 + H 15.45200 14.06400 -9.41700 + H 16.12800 14.37600 -10.73500 + O 15.22100 13.38800 -7.72100 + H 14.70000 12.64400 -8.02300 + H 15.65600 13.07100 -6.92900 + O 13.78200 9.37600 -3.38200 + H 12.97900 9.38900 -2.86100 + H 14.34600 10.03000 -2.97000 + O 14.96400 7.86300 -16.78100 + H 15.67600 8.47200 -16.97800 + H 15.37300 7.18100 -16.24900 + O 16.00700 5.58600 -15.45900 + H 15.11900 5.31400 -15.22500 + H 16.47100 5.63400 -14.62300 + O 15.39800 8.48700 -10.36300 + H 15.93600 7.72600 -10.57900 + H 15.93500 9.23900 -10.61300 + O 13.42200 11.46900 -8.53000 + H 12.72000 11.76500 -7.95100 + H 13.02400 11.45100 -9.40000 + O 14.41200 6.41800 -19.02200 + H 14.10600 7.06400 -19.65900 + H 14.51300 6.91300 -18.20800 + O 14.04300 2.93200 -21.26100 + H 13.15500 2.96900 -21.61900 + H 14.31000 2.02000 -21.37400 + O 13.38200 4.73800 -15.02400 + H 13.33300 3.88500 -15.45500 + H 12.93900 4.61000 -14.18500 + O -2.63800 5.60900 -11.56700 + H -2.61800 5.97600 -12.45100 + H -1.74200 5.70600 -11.24600 + O 13.46900 6.57400 -6.29700 + H 13.90000 6.02400 -6.95200 + H 13.37700 7.42600 -6.72300 + O 15.32900 3.57400 -6.27800 + H 14.55800 3.31800 -5.77100 + H 15.92500 2.82900 -6.20100 + O 14.29300 0.14400 -21.65900 + H 15.17200 -0.08700 -21.36100 + H 13.71700 -0.13900 -20.94900 + O 13.19200 2.48600 -16.63600 + H 13.51300 1.59500 -16.49400 + H 13.80000 2.86100 -17.27200 + O 13.78900 2.69400 -11.17800 + H 14.74000 2.75400 -11.08100 + H 13.45100 2.78200 -10.28700 + O 12.99300 3.17200 -8.57400 + H 12.10000 3.45600 -8.37900 + H 13.51800 3.97000 -8.50800 + O -1.82200 1.24200 -5.85800 + H -1.07300 1.00500 -5.31000 + H -2.49500 0.59900 -5.63400 + O 15.73000 -2.48700 -4.05600 + H 16.56400 -2.92300 -4.23000 + H 15.65400 -1.83100 -4.74900 + O 13.65100 -0.15400 -16.21200 + H 13.75000 -0.48400 -15.31900 + H 13.06700 -0.78200 -16.63600 + O -1.12700 -0.11600 -12.34400 + H -0.67600 -0.95800 -12.40600 + H -0.43500 0.53600 -12.45200 + O -3.25000 3.08800 -12.57700 + H -2.41900 3.08200 -13.05100 + H -3.23900 3.90900 -12.08400 + O 15.30600 -0.90900 -6.32700 + H 14.55300 -0.41500 -6.65200 + H 15.32100 -1.70400 -6.86100 + O 16.87300 -0.57000 -20.98400 + H 17.74700 -0.94000 -21.10800 + H 16.97900 0.06200 -20.27300 + O -1.65600 0.56000 -17.40700 + H -2.51700 0.66500 -17.00200 + H -1.74300 0.97600 -18.26500 + O 11.39500 14.95700 -14.52200 + H 12.10100 14.91700 -15.16600 + H 11.76700 15.44700 -13.78800 + O -2.81900 -2.87100 -8.65800 + H -3.44700 -2.70900 -7.95400 + H -2.92300 -2.12500 -9.24900 + O -0.74600 12.50500 -5.31100 + H -1.64100 12.38000 -5.62400 + H -0.72300 12.06100 -4.46400 + O 3.03000 12.74900 -8.32200 + H 2.72400 13.63300 -8.52800 + H 3.94900 12.74500 -8.58800 + O 5.65900 13.73600 -5.43700 + H 5.90300 13.16400 -4.71000 + H 6.45200 14.23800 -5.62500 + O 8.05200 14.95800 -6.04100 + H 8.39300 14.42000 -6.75500 + H 8.80400 15.09600 -5.46500 + O 15.23600 -3.39100 -7.63400 + H 15.39300 -3.04000 -8.51000 + H 15.58800 -4.28100 -7.66500 + O -0.28900 15.42000 -8.74800 + H -1.09700 15.30600 -9.24900 + H -0.37600 16.28500 -8.34800 + O 2.42000 15.31800 -9.04700 + H 1.47800 15.35800 -8.88500 + H 2.52500 15.64600 -9.94000 + O 6.90900 15.18900 -9.82900 + H 6.34500 15.44400 -10.55900 + H 6.86400 15.92700 -9.22100 + O 8.83600 13.79000 -8.45600 + H 8.25300 14.27700 -9.03900 + H 9.69100 14.20400 -8.57300 + O 11.41500 14.68300 -8.49000 + H 12.02700 15.28300 -8.06400 + H 11.77500 14.55600 -9.36800 + O -0.22400 13.98200 -12.42600 + H -0.31500 13.62700 -11.54200 + H 0.53500 13.52600 -12.79000 + O 2.05800 12.64100 -13.24800 + H 2.04300 11.68600 -13.31500 + H 2.84400 12.83300 -12.73700 + O 4.44600 12.94500 -11.84900 + H 4.87400 12.21300 -12.29200 + H 4.73800 12.87900 -10.94000 + O 6.31500 -2.26400 -13.75600 + H 5.80900 -2.46500 -12.96900 + H 7.22500 -2.25100 -13.46000 + O -0.10900 -2.98700 -15.30500 + H -1.03300 -3.23400 -15.34100 + H 0.36300 -3.79900 -15.48900 + O -1.90600 -1.95700 -18.68500 + H -1.07300 -2.05300 -19.14800 + H -1.78600 -1.18400 -18.13400 + O 6.07200 -4.76300 -15.09300 + H 6.65200 -4.59100 -15.83500 + H 6.04500 -3.93500 -14.61300 + O 8.90600 -2.48800 -12.76500 + H 9.82900 -2.24100 -12.83200 + H 8.92000 -3.43800 -12.65500 + O 11.69300 -1.75300 -17.39100 + H 10.96500 -1.15900 -17.21200 + H 11.37300 -2.61400 -17.12100 + O 0.57800 -2.09400 -19.81500 + H 0.83400 -2.27600 -20.71900 + H 1.30200 -1.57800 -19.46000 + O 5.68000 -1.01700 -18.72800 + H 6.32200 -1.35000 -18.10200 + H 5.60200 -0.08600 -18.52000 + O 6.52000 15.32400 -20.26200 + H 7.47100 15.40600 -20.33400 + H 6.28400 14.67400 -20.92300 + O 13.53400 14.63300 -16.44100 + H 13.73700 13.71300 -16.61300 + H 13.46800 15.03000 -17.30900 + O 0.34700 -2.34600 -5.65700 + H -0.27600 -2.84000 -5.12300 + H 1.18600 -2.45700 -5.21000 + O 2.38600 -2.75800 -3.85400 + H 3.21000 -3.04600 -4.24800 + H 2.09700 -3.50600 -3.33100 + O 0.87000 16.03800 20.82600 + H 1.32000 16.50600 21.53000 + H 0.09800 16.57200 20.63900 + O 10.73400 -2.94400 -4.53300 + H 10.64200 -3.43600 -3.71700 + H 11.63400 -3.11100 -4.81300 + O 13.22600 12.83200 -4.79700 + H 12.63900 12.55400 -5.50000 + H 13.11900 13.78300 -4.75600 + O 18.70900 4.46700 -9.52900 + H 18.82600 3.78700 -8.86600 + H 19.59700 4.66900 -9.82500 + O 19.70500 9.60000 -11.55000 + H 19.74800 8.68100 -11.81400 + H 20.27500 9.65600 -10.78300 + O 17.11500 3.74800 -17.42400 + H 16.89300 4.33900 -16.70400 + H 16.31500 3.69300 -17.94500 + O 17.46900 5.54600 -13.02200 + H 17.78200 4.64200 -12.99500 + H 18.25100 6.07500 -12.86900 + O 19.98700 -1.14400 -13.99800 + H 20.93900 -1.13900 -14.10200 + H 19.74500 -2.06900 -14.05700 + O 18.26800 2.92500 -12.70300 + H 19.11900 2.65000 -12.36300 + H 18.02900 2.24500 -13.33300 + O 20.79000 -0.63100 -10.53700 + H 20.72800 0.14300 -11.09700 + H 20.08700 -0.52100 -9.89700 + O 20.67400 1.67600 -12.10900 + H 21.43100 2.23100 -11.92100 + H 20.74400 1.48400 -13.04400 + O 20.91600 10.39700 -8.98000 + H 21.17700 11.16800 -8.47700 + H 20.07400 10.13900 -8.60600 + O 17.01500 9.49100 -17.68000 + H 16.88100 9.68400 -18.60800 + H 17.16500 10.34500 -17.27600 + O 21.56200 5.73100 -13.90500 + H 21.58300 5.13800 -14.65700 + H 22.44300 6.10200 -13.86700 + O 21.39700 4.87600 -10.35800 + H 21.84800 4.27000 -10.94600 + H 21.97700 5.63600 -10.30700 + O 19.85800 5.93200 -6.66300 + H 19.81800 6.82400 -6.31600 + H 18.97900 5.76700 -7.00300 + O 21.29200 11.90800 -4.98500 + H 21.37600 12.16800 -5.90200 + H 20.35600 11.74800 -4.86600 + O 19.68400 4.82300 -17.96700 + H 19.40600 5.73100 -18.09200 + H 18.86800 4.33700 -17.84900 + O 16.36100 8.29200 -7.72600 + H 15.92000 8.27200 -8.57500 + H 16.53400 7.37300 -7.52300 + O 17.18100 10.51900 -10.82400 + H 18.01300 10.21900 -11.18800 + H 17.42500 11.19500 -10.19200 + O 19.17100 13.14800 -15.40600 + H 19.16500 12.65900 -14.58300 + H 18.48900 12.73500 -15.93600 + O 18.58500 9.98000 -7.55200 + H 18.98700 9.58000 -6.78100 + H 17.78500 9.47400 -7.69600 + O 21.11100 -3.07400 -11.88100 + H 20.44700 -3.30000 -12.53100 + H 20.88300 -2.18700 -11.60200 + O 21.77400 2.97200 -8.25300 + H 20.87600 2.66500 -8.13500 + H 21.71800 3.61400 -8.96000 + O 21.03300 1.59700 -14.85200 + H 20.21500 1.31700 -15.26200 + H 21.22200 2.44700 -15.25000 + O 19.07500 2.31400 -7.83800 + H 18.37500 2.29400 -7.18500 + H 19.08700 1.43000 -8.20400 + O 17.38100 1.12400 -18.85700 + H 17.99900 0.53500 -18.42500 + H 17.42500 1.93400 -18.34800 + O 22.01500 4.64600 -19.49800 + H 21.17000 4.72800 -19.05500 + H 21.81500 4.78600 -20.42300 + O 19.58700 8.43900 -5.43200 + H 20.35000 8.43700 -4.85400 + H 18.83600 8.42300 -4.83900 + O 19.86900 6.78200 -12.07000 + H 20.40800 6.53900 -12.82300 + H 20.19200 6.23000 -11.35700 + O 16.93800 5.77200 -6.69000 + H 17.23700 5.74100 -5.78200 + H 16.30200 5.06000 -6.75800 + O 16.89700 5.87300 -20.10100 + H 17.59700 6.29100 -19.60100 + H 16.10000 6.07100 -19.60900 + O 18.80100 -0.31200 -8.59100 + H 17.86200 -0.24500 -8.76600 + H 18.86300 -0.90100 -7.83900 + O 18.17100 12.41700 -8.94000 + H 18.32100 11.67900 -8.35000 + H 18.30100 13.19200 -8.39300 + O 18.65800 7.34500 -18.51800 + H 18.17800 7.96900 -17.97400 + H 19.10400 7.88700 -19.16900 + O 19.50100 9.11500 -20.45800 + H 19.70900 10.04900 -20.48400 + H 20.12900 8.71200 -21.05700 + O 19.15900 -3.74800 -13.78100 + H 18.42700 -3.73500 -14.39800 + H 18.98800 -4.50500 -13.22100 + O 17.01900 12.20000 -16.96800 + H 16.07800 12.19900 -16.79100 + H 17.11400 12.73400 -17.75700 + O 17.30000 14.40600 -3.74700 + H 17.91400 13.98300 -3.14600 + H 16.88400 13.68200 -4.21400 + O -21.98700 11.64300 -2.30000 + H -21.17900 11.14200 -2.40700 + H -22.25800 11.85500 -3.19400 + O 17.34600 8.21400 -3.81600 + H 17.29500 7.26700 -3.94700 + H 16.68200 8.57300 -4.40300 + O 21.33100 7.90600 21.77700 + H 21.55200 6.97600 21.82300 + H 21.88200 8.24800 21.07200 + O 17.90900 5.45400 -4.12200 + H 18.38600 4.67000 -4.39500 + H 18.02300 5.48600 -3.17200 + O 21.75500 4.16700 -5.77000 + H 21.13300 4.84700 -6.03100 + H 21.94000 3.68900 -6.57800 + O 19.43500 3.11800 -4.53500 + H 19.39200 2.17900 -4.71300 + H 20.23500 3.40900 -4.97100 + O 19.18300 0.33900 -4.76700 + H 19.24900 -0.40200 -5.37000 + H 19.56500 0.01900 -3.95000 + O 16.84300 9.92000 -20.46000 + H 17.64900 9.40400 -20.42200 + H 16.95100 10.47800 -21.23000 + O 15.73500 15.25900 -14.87700 + H 15.01700 15.07600 -15.48300 + H 15.78700 14.47900 -14.32400 + O 15.86500 13.03600 -13.30200 + H 15.53700 12.14400 -13.19400 + H 16.47900 13.15500 -12.57700 + O 16.58100 5.98000 -10.42200 + H 16.71700 5.83000 -11.35700 + H 17.35100 5.59600 -10.00200 + O 16.97500 1.33800 -6.10100 + H 17.66000 1.07400 -5.48600 + H 16.41900 0.56400 -6.18800 + O 16.54300 2.74900 -10.57200 + H 17.06100 2.76000 -11.37700 + H 16.95000 3.41300 -10.01600 + O 16.10800 -2.32700 -10.01400 + H 16.09900 -1.37700 -9.89900 + H 15.68500 -2.47100 -10.86000 + O 18.61700 11.54600 -4.45600 + H 17.79500 11.74500 -4.90500 + H 18.34700 11.17200 -3.61700 + O -22.01700 12.59400 -7.59700 + H -22.32000 13.46300 -7.85900 + H -21.09200 12.57700 -7.84300 + O 18.54200 14.87400 -7.61900 + H 17.83600 15.47500 -7.85400 + H 18.82800 15.16600 -6.75400 + O 21.17200 15.08500 -8.48100 + H 20.24100 15.07400 -8.25500 + H 21.19100 15.32100 -9.40800 + O 17.82700 13.71200 -11.37000 + H 17.98300 13.33700 -10.50400 + H 18.39300 13.20800 -11.95500 + O 19.49500 12.09900 -12.83800 + H 19.63100 11.20300 -12.53200 + H 20.33900 12.53000 -12.70300 + O 17.08300 13.68100 -19.31700 + H 16.22300 13.61500 -19.73100 + H 17.20100 14.61700 -19.15500 + O 19.17800 -2.05200 -6.35600 + H 18.63200 -2.61500 -5.80600 + H 20.07600 -2.27700 -6.11300 + O 21.26200 -3.00800 -4.56000 + H 22.19300 -3.22200 -4.62000 + H 20.88300 -3.72900 -4.05700 + O 19.22200 12.12400 -20.35500 + H 18.49400 12.61100 -19.96900 + H 20.00600 12.54200 -20.00000 + O -3.11200 18.10100 -10.39000 + H -3.11000 18.91800 -9.89200 + H -2.53500 18.27300 -11.13500 + O -10.29700 21.85200 -11.98200 + H -10.18200 20.90200 -12.01700 + H -9.77400 22.18400 -12.71200 + O -16.68900 19.31600 -14.41600 + H -16.31700 19.77400 -15.17000 + H -17.33000 18.71400 -14.79400 + O -8.01700 20.36700 -15.78100 + H -7.69800 19.77800 -15.09700 + H -7.29500 20.42100 -16.40700 + O -3.10700 20.47100 -9.08000 + H -3.76900 21.00600 -9.51700 + H -2.93500 20.92600 -8.25600 + O -17.50100 18.89200 -9.88800 + H -17.79600 19.34100 -10.68000 + H -18.19800 19.04800 -9.25000 + O -7.31000 19.18900 -13.33100 + H -8.13200 19.27200 -12.84800 + H -6.84400 20.00600 -13.15300 + O -18.12400 20.18300 -12.20700 + H -17.86900 21.10100 -12.11100 + H -17.69000 19.89600 -13.01000 + O -5.61600 17.05900 -9.90200 + H -6.04000 16.98200 -10.75600 + H -4.79100 17.50900 -10.08200 + O -13.90100 15.56400 -11.45700 + H -14.09100 14.62700 -11.50400 + H -14.76000 15.97900 -11.37600 + O -16.45900 16.68600 -11.22500 + H -17.14600 16.49500 -11.86400 + H -16.74900 17.49100 -10.79600 + O -11.86800 16.96200 -7.60000 + H -11.36300 16.58000 -6.88200 + H -12.77900 16.89900 -7.31300 + O -9.05300 20.41000 -8.17400 + H -9.50900 20.33800 -7.33500 + H -9.32700 21.25800 -8.52400 + O -12.46600 18.40800 -20.07800 + H -13.25600 18.90200 -20.29700 + H -12.73600 17.82900 -19.36500 + O -14.80000 19.72100 -20.54600 + H -15.62000 19.54500 -20.08600 + H -15.05700 20.24900 -21.30200 + O 21.86700 19.43600 -14.73200 + H 22.33500 19.03300 -14.00000 + H 21.37900 18.71600 -15.13000 + O -21.05700 19.35400 -19.81900 + H -20.26500 18.82600 -19.91700 + H -20.80600 20.22300 -20.13100 + O -14.43700 19.45200 -12.77100 + H -15.27600 19.35500 -13.22300 + H -13.92700 18.69100 -13.04700 + O -19.65900 20.50100 -15.59300 + H -20.53900 20.22000 -15.34200 + H -19.19500 19.68800 -15.79300 + O -15.49700 20.48300 -16.62800 + H -14.58100 20.20900 -16.67900 + H -15.45800 21.40100 -16.35900 + O -9.83100 19.14000 -12.14300 + H -10.03400 18.77100 -11.28300 + H -10.04400 18.44000 -12.76100 + O -18.51000 17.98500 -16.04300 + H -18.97700 17.20100 -15.75200 + H -18.30100 17.81300 -16.96100 + O -13.09000 19.61100 -10.27100 + H -13.44700 19.59600 -11.15900 + H -12.25300 19.15200 -10.34100 + O -10.51000 20.14900 -20.89100 + H -11.10900 19.58600 -20.40100 + H -10.19300 20.78200 -20.24700 + O -9.47300 -21.81200 -19.16600 + H -8.55100 -21.76200 -19.41600 + H -9.77400 -20.90300 -19.17400 + O -19.23500 19.43000 -7.69300 + H -19.77200 19.94500 -7.09000 + H -19.73400 18.62500 -7.83100 + O -10.60800 18.51500 -9.55600 + H -9.94200 19.03700 -9.10900 + H -10.97300 17.95400 -8.87200 + O -10.31700 20.25200 -17.26500 + H -10.14400 20.89900 -17.94800 + H -9.56600 20.31900 -16.67500 + O -12.88500 19.49700 -16.49900 + H -12.06600 19.83300 -16.86100 + H -12.68100 19.29800 -15.58500 + O -12.46200 21.13500 -8.03500 + H -11.87500 20.64400 -7.46000 + H -12.62600 20.54400 -8.77000 + O -9.77800 17.53200 -16.75700 + H -10.01000 18.40800 -17.06400 + H -8.98600 17.30800 -17.24500 + O -16.62000 18.69000 -18.54800 + H -16.51400 19.41500 -17.93100 + H -15.84400 18.14600 -18.41300 + O -6.25000 20.60300 -8.11300 + H -7.19500 20.44900 -8.13200 + H -6.01600 20.77100 -9.02600 + O -19.22500 -21.82700 20.34700 + H -18.44000 -21.40000 20.69000 + H -19.74200 -22.04000 21.12500 + O -18.83900 16.38100 -12.65000 + H -19.12200 16.09600 -13.51900 + H -19.01600 15.63100 -12.08200 + O -16.71600 20.51600 -7.03000 + H -16.15000 19.88700 -7.47700 + H -17.60300 20.27100 -7.29400 + O -15.12100 18.95800 -8.56800 + H -14.40100 19.15400 -9.16700 + H -15.88400 18.85500 -9.13500 + O -17.20100 20.83000 -4.36200 + H -17.93200 20.24700 -4.15900 + H -17.04900 20.70800 -5.29900 + O -14.61600 16.65500 -7.16300 + H -14.78200 17.52700 -7.52000 + H -15.18300 16.07500 -7.67100 + O -12.68300 20.46500 -3.93200 + H -12.11600 20.14200 -3.23200 + H -13.54500 20.10200 -3.72900 + O -10.82400 20.04900 -6.02100 + H -11.47700 20.25000 -5.35100 + H -10.14900 19.55400 -5.55700 + O -8.92800 18.79400 -4.43200 + H -8.78600 17.86500 -4.25200 + H -8.96000 19.20300 -3.56700 + O -19.12600 18.83300 -4.08700 + H -19.12600 17.98400 -4.52900 + H -18.81600 18.64400 -3.20200 + O -14.69100 15.52200 -4.63000 + H -14.09500 14.78800 -4.77600 + H -14.74300 15.96200 -5.47800 + O -13.70400 17.17400 -2.65600 + H -13.99100 16.54400 -3.31800 + H -14.26100 17.94000 -2.79700 + O -4.67700 19.87800 -21.35800 + H -4.26300 19.14400 -21.81300 + H -5.01800 19.49700 -20.54800 + O -6.10200 20.72900 -17.72100 + H -5.97600 19.91500 -18.21000 + H -5.27000 21.19300 -17.81000 + O -5.34300 21.65200 -10.49500 + H -5.59000 21.49000 -11.40600 + H -5.59300 22.56200 -10.33700 + O -20.70400 20.53300 -5.49200 + H -20.19800 19.92400 -4.95300 + H -21.59000 20.17100 -5.49500 + O -3.26700 16.34800 -2.92100 + H -2.54300 15.92600 -3.38400 + H -2.84200 16.93500 -2.29600 + O -5.49900 18.42400 -19.09200 + H -4.75900 17.83600 -18.94500 + H -6.27200 17.88500 -18.92500 + O -20.42900 18.63500 -12.68300 + H -19.95400 17.80800 -12.60400 + H -19.77900 19.30700 -12.47400 + O -4.68600 19.32000 -6.27800 + H -5.19100 19.74000 -6.97400 + H -4.68700 18.39100 -6.50800 + O 21.55500 17.23400 -18.39700 + H 21.02100 17.15000 -19.18700 + H 22.09200 18.01100 -18.55200 + O -20.93100 17.23200 -8.23900 + H -21.50500 16.48000 -8.09200 + H -21.41800 17.79100 -8.84500 + O -4.95600 16.67100 -7.16800 + H -5.30800 16.87600 -8.03400 + H -4.60800 15.78300 -7.25400 + O -12.94800 17.23300 -13.45300 + H -13.15800 16.61300 -12.75500 + H -12.00200 17.14800 -13.57300 + O -6.75600 16.61600 -12.41800 + H -6.81900 17.47700 -12.83100 + H -7.49000 16.12100 -12.78200 + O -19.67600 15.60800 -15.20000 + H -20.51400 15.15800 -15.30700 + H -19.02100 14.91200 -15.25600 + O -20.24100 15.32900 -19.02100 + H -19.73800 16.07000 -19.35900 + H -21.02800 15.72300 -18.64500 + O -13.80700 15.83200 -15.71900 + H -12.95100 15.51100 -16.00300 + H -13.62800 16.31000 -14.90900 + O -10.25200 17.15300 -14.03000 + H -10.08400 17.34400 -14.95200 + H -9.71300 16.38400 -13.84200 + O -7.60400 16.73400 -18.28300 + H -7.96400 16.23400 -19.01600 + H -7.02200 16.11900 -17.83600 + O -18.83500 17.61000 -19.90600 + H -18.55600 17.26400 -20.75400 + H -18.04200 17.99000 -19.52900 + O -14.17300 17.46100 -17.88600 + H -14.19200 16.82300 -17.17200 + H -13.75900 18.23500 -17.50400 + O -18.96200 16.20700 -4.95600 + H -19.60700 15.60800 -4.57900 + H -18.19800 16.12800 -4.38400 + O -16.99400 15.97400 -3.03800 + H -16.18600 15.70000 -3.47200 + H -17.30200 15.18900 -2.58500 + O -8.40000 16.07800 -3.88100 + H -8.50800 15.66200 -3.02600 + H -7.45400 16.08100 -4.02900 + O 16.05500 18.68300 -9.92000 + H 16.25400 19.43800 -9.36500 + H 16.67200 18.75100 -10.64800 + O 8.61500 22.27800 -12.28900 + H 8.47400 21.36500 -12.03900 + H 9.17000 22.23200 -13.06700 + O 2.91100 20.12500 -13.82000 + H 3.14300 21.01000 -14.10300 + H 2.26500 19.83000 -14.46200 + O 11.24900 19.86200 -15.32600 + H 11.22900 19.44100 -14.46700 + H 12.05200 19.53700 -15.73400 + O 16.99000 20.63200 -8.16400 + H 16.77600 21.51100 -8.47700 + H 16.90300 20.68800 -7.21300 + O 1.21100 18.49600 -9.76100 + H 0.92900 19.18200 -10.36600 + H 0.46400 18.37000 -9.17600 + O 10.97000 19.02300 -12.64900 + H 10.04500 19.07200 -12.40800 + H 11.34700 19.83800 -12.31900 + O 1.05000 20.40300 -11.74300 + H 1.20800 21.31300 -11.49000 + H 1.72500 20.21400 -12.39400 + O 13.62100 17.40100 -9.79200 + H 13.25400 17.14500 -10.63800 + H 14.44300 17.84000 -10.01000 + O 4.99300 15.70100 -11.80700 + H 4.81100 14.77800 -11.98500 + H 4.12900 16.10400 -11.72700 + O 13.19400 15.77700 -18.99200 + H 12.37100 16.07900 -18.60700 + H 13.54100 16.54400 -19.44800 + O 2.41400 16.69000 -11.46400 + H 1.75800 16.63800 -12.15900 + H 2.09000 17.37600 -10.87900 + O 6.92900 16.93300 -7.66800 + H 7.30500 16.35600 -7.00300 + H 5.98900 16.92700 -7.48700 + O 10.43200 20.04800 -7.83600 + H 9.92500 20.36400 -7.08800 + H 10.69600 20.84300 -8.29900 + O 6.16200 19.15200 -21.52000 + H 5.37600 19.51300 -21.92900 + H 5.97200 19.16200 -20.58200 + O 3.56100 19.94000 21.41300 + H 3.24100 20.04200 22.30900 + H 2.76800 19.84400 20.88400 + O -2.41900 21.19300 -13.59800 + H -2.25100 20.37800 -13.12500 + H -2.06600 21.03900 -14.47400 + O -2.54700 19.20500 -17.74100 + H -2.73500 18.28700 -17.93700 + H -3.23700 19.69300 -18.19200 + O 5.52200 19.11600 -14.62000 + H 4.67300 19.27200 -14.20700 + H 5.98400 18.54200 -14.00800 + O -1.19500 20.93000 -16.05500 + H -1.70800 20.39100 -16.65700 + H -0.38100 20.44100 -15.93100 + O 2.72600 21.19400 -17.55700 + H 3.60000 21.16000 -17.16800 + H 2.53600 22.12900 -17.63700 + O 8.33400 19.45900 -11.84400 + H 8.24400 19.27500 -10.90900 + H 7.80200 18.78900 -12.27300 + O 1.15200 19.48500 -15.81800 + H 0.97400 18.55500 -15.95800 + H 1.60800 19.76400 -16.61200 + O 4.70700 21.59900 -9.37000 + H 4.39400 22.33300 -8.84200 + H 5.55800 21.37800 -8.99200 + O 9.00800 19.39700 -21.14200 + H 8.06000 19.29200 -21.21900 + H 9.12400 19.97900 -20.39100 + O 9.57900 21.18200 -19.03800 + H 10.51100 21.38400 -19.12300 + H 9.14200 22.03000 -19.11400 + O -0.75100 18.03900 -7.83100 + H -1.50000 18.63200 -7.88700 + H -0.58500 17.95100 -6.89200 + O 8.21600 18.89500 -9.13600 + H 9.06500 19.05300 -8.72300 + H 7.84000 18.16700 -8.64100 + O 9.34800 19.10400 -17.17700 + H 9.36000 19.79800 -17.83600 + H 9.94000 19.41400 -16.49200 + O 5.38800 21.00400 -16.63400 + H 5.99400 21.69200 -16.36100 + H 5.44000 20.34800 -15.93900 + O 6.93600 20.89900 -7.74400 + H 7.47200 20.76300 -6.96300 + H 7.26200 20.25700 -8.37400 + O 6.85800 17.93100 -16.86200 + H 6.47600 18.28200 -16.05700 + H 7.73900 18.30400 -16.89200 + O 2.97700 19.64600 -19.83000 + H 2.58600 20.20600 -19.15900 + H 3.84600 19.43500 -19.49000 + O 13.08800 19.19100 -7.68200 + H 12.14700 19.36600 -7.68900 + H 13.22600 18.59500 -8.41800 + O 0.99800 -21.71100 20.37200 + H 1.02900 -22.64600 20.17200 + H 0.48300 -21.33000 19.66100 + O 0.20000 16.66500 -13.18000 + H 0.29000 16.71300 -14.13200 + H -0.03800 15.75400 -13.00800 + O 4.44400 20.74800 -6.40900 + H 3.99200 20.15200 -7.00600 + H 5.33300 20.80800 -6.75600 + O 3.49200 19.25100 -8.49500 + H 3.83000 19.97100 -9.02700 + H 2.68900 18.98000 -8.94000 + O 2.39300 20.84900 -4.53300 + H 1.53600 20.64700 -4.90900 + H 2.99700 20.80600 -5.27500 + O 4.16600 16.79900 -7.44400 + H 3.94100 17.67200 -7.76600 + H 3.61400 16.20600 -7.95400 + O 6.57000 20.02400 -3.58300 + H 6.95500 19.53000 -2.86000 + H 5.63700 19.81300 -3.54800 + O 8.51400 20.13600 -5.62700 + H 7.85100 20.17200 -4.93800 + H 9.10800 19.43800 -5.35200 + O 10.11900 18.17900 -4.47800 + H 10.09200 17.23000 -4.59600 + H 9.83500 18.31700 -3.57400 + O -0.38400 20.27800 -4.95100 + H -0.42500 19.32300 -4.90800 + H -0.53100 20.56500 -4.05000 + O 3.92100 15.81400 -4.87000 + H 4.35100 14.97100 -5.01400 + H 3.97200 16.25700 -5.71700 + O 4.79600 16.66300 -2.37900 + H 4.57900 16.19800 -3.18700 + H 4.41200 17.53300 -2.49200 + O 11.86500 19.75700 -21.24100 + H 12.44200 19.05700 -20.93500 + H 10.98500 19.38500 -21.18400 + O 12.25200 21.83200 -19.44900 + H 12.22400 21.05300 -20.00500 + H 12.88100 22.41100 -19.87800 + O 11.57300 -21.81100 -9.35100 + H 11.60100 -22.13400 -10.25100 + H 11.72000 -20.86900 -9.43000 + O -2.58900 21.15900 -6.40600 + H -1.80600 20.86800 -5.93800 + H -3.26200 20.52300 -6.16400 + O 15.40100 16.57000 -3.82600 + H 16.18600 16.03300 -3.93900 + H 15.48700 17.26200 -4.48200 + O 13.28100 18.28700 -16.47000 + H 13.58200 17.87500 -15.66000 + H 12.62400 17.68400 -16.81700 + O -1.23300 18.91700 -12.33400 + H -0.82800 18.11600 -12.66700 + H -0.49600 19.49400 -12.13200 + O 14.51900 18.84500 -5.36200 + H 13.85200 19.15200 -4.74800 + H 14.08400 18.85400 -6.21400 + O 13.88200 18.04200 -20.43700 + H 14.10000 17.59000 -21.25200 + H 14.60300 18.65700 -20.30300 + O -2.78800 15.37900 -9.84900 + H -3.29100 15.07100 -9.09400 + H -3.00300 16.30900 -9.91600 + O 13.91600 15.83300 -7.53800 + H 13.85800 16.27400 -8.38600 + H 14.35400 15.00400 -7.72800 + O 6.91700 17.30400 -13.00300 + H 6.22800 16.79200 -12.57900 + H 7.54900 16.65200 -13.30600 + O 12.48400 16.70700 -12.42300 + H 13.21400 16.99100 -12.97300 + H 11.86100 17.43300 -12.45700 + O 14.48000 17.56700 -14.06700 + H 15.06200 16.83900 -14.28500 + H 15.06700 18.31400 -13.94600 + O 0.50500 16.78800 -15.98500 + H -0.30800 16.62700 -16.46300 + H 0.95700 15.94400 -15.98900 + O 6.46600 13.90100 -15.21800 + H 6.60000 14.33200 -16.06200 + H 7.04300 14.36900 -14.61500 + O 8.56500 15.14300 -13.74600 + H 9.45500 15.17900 -14.09700 + H 8.56800 14.38500 -13.16200 + O 11.09200 16.86500 -17.52600 + H 10.43700 17.56300 -17.51100 + H 10.58100 16.05600 -17.49000 + O 2.62600 17.34400 -21.32200 + H 3.19000 17.72400 -21.99600 + H 2.50400 18.04900 -20.68600 + O 5.52200 19.32300 -18.83800 + H 5.94600 18.62300 -18.34100 + H 5.48700 20.06200 -18.23100 + O -0.30600 17.52600 -5.10200 + H -0.81800 16.74000 -4.91200 + H 0.50500 17.41000 -4.60700 + O 1.75100 16.79100 -3.40600 + H 2.51600 16.54900 -3.92800 + H 1.48200 15.97700 -2.98000 + O 10.25100 15.40400 -4.37900 + H 10.13600 15.02900 -3.50600 + H 11.19700 15.38700 -4.52500 + O 19.65300 20.87600 -14.13000 + H 20.50100 20.50700 -14.37400 + H 19.04500 20.53500 -14.78600 + O 21.16800 18.58500 -9.80300 + H 21.04000 19.32200 -10.40000 + H 20.39800 18.60100 -9.23400 + O 20.35600 20.66800 -11.47600 + H 20.82600 21.48300 -11.30000 + H 20.11400 20.72500 -12.40000 + O 21.13200 16.07300 -11.04300 + H 20.32000 16.17500 -11.54100 + H 21.37300 16.96400 -10.79200 + O 16.29800 19.67800 -13.78600 + H 16.89400 19.48700 -13.06200 + H 16.86600 19.74800 -14.55300 + O 15.99600 19.77000 -20.00400 + H 16.75700 19.32800 -20.38200 + H 16.25700 20.68900 -19.94400 + O 17.98000 19.72100 -16.08100 + H 17.26300 19.33500 -16.58500 + H 18.69800 19.09400 -16.16800 + O 20.01600 17.81500 -16.09800 + H 19.54000 17.04900 -15.77700 + H 20.51200 17.49100 -16.85100 + O 18.89000 18.59200 -8.25200 + H 18.35400 19.38400 -8.27100 + H 18.27600 17.89100 -8.03300 + O 20.73900 17.99000 -21.01300 + H 21.31100 18.66400 -20.64500 + H 21.31200 17.47100 -21.57700 + O 18.06300 -21.03600 20.41800 + H 18.31300 -21.95500 20.50800 + H 17.11700 -21.02800 20.56700 + O 18.71100 16.26400 -12.33900 + H 18.67100 16.16100 -13.29000 + H 18.29500 15.47300 -11.99400 + O 19.71900 20.73700 -3.28800 + H 18.97600 21.02400 -2.75700 + H 19.37000 20.02100 -3.82000 + O 18.48100 18.49700 -4.40500 + H 18.73800 17.80600 -5.01500 + H 18.55400 18.09300 -3.54000 + O 16.74400 20.43000 -5.37200 + H 17.41400 19.81700 -5.06900 + H 15.93400 19.92100 -5.36800 + O 18.06600 18.92200 -11.79200 + H 18.39100 18.03500 -11.94500 + H 18.84600 19.42800 -11.56600 + O 15.88000 18.61100 -17.50200 + H 14.98400 18.53400 -17.17500 + H 15.79100 19.05400 -18.34600 + O 16.63100 16.84900 -7.93300 + H 15.78800 16.62700 -7.53600 + H 16.40400 17.37600 -8.70000 + O 18.45000 15.77200 -15.05700 + H 17.49700 15.70400 -15.12700 + H 18.76800 14.89100 -15.25800 + O 17.12400 16.42800 -18.95900 + H 17.45100 16.99600 -19.65600 + H 16.73300 17.02800 -18.32400 + O 17.97600 18.15300 -21.02300 + H 17.68800 17.88700 -21.89700 + H 18.93100 18.10100 -21.05900 + O 19.19300 15.85000 -5.11800 + H 18.51100 15.37400 -4.64500 + H 19.98500 15.71600 -4.59700 + O 20.98200 15.22100 -3.00800 + H 21.82600 14.86600 -3.28700 + H 20.49400 14.45900 -2.69400 + O -9.50200 -11.04600 2.64700 + H -9.79700 -10.13500 2.66300 + H -10.22000 -11.52300 2.23000 + O -10.60900 -11.95900 7.28300 + H -10.35100 -12.53500 8.00300 + H -11.22600 -12.48000 6.76900 + O -18.32900 -13.79700 9.56800 + H -18.39400 -14.57500 10.12200 + H -18.13900 -13.08400 10.17700 + O -6.72300 -8.98400 5.91600 + H -7.17000 -9.77100 6.22700 + H -6.84500 -8.34500 6.61900 + O -15.45200 -7.88800 6.11600 + H -15.49600 -8.77400 6.47700 + H -14.51900 -7.67600 6.11100 + O -20.43000 -13.92600 1.66000 + H -20.30600 -13.60600 2.55400 + H -21.02400 -14.67100 1.75200 + O -20.19300 -13.29500 4.30700 + H -20.06900 -14.11200 4.79100 + H -19.59900 -12.67400 4.73000 + O -9.60000 -4.71700 6.83200 + H -9.26000 -5.27600 6.13400 + H -9.73000 -5.31000 7.57300 + O -11.75800 -9.41300 7.61000 + H -11.36000 -10.23900 7.33600 + H -11.26200 -9.15300 8.38700 + O -2.76500 -19.14000 8.80200 + H -2.90600 -18.27400 9.18200 + H -2.02800 -19.02200 8.20200 + O -10.08100 -15.48900 6.67300 + H -10.08600 -16.42100 6.89000 + H -9.70200 -15.44900 5.79500 + O -15.47600 -17.61600 4.61500 + H -15.46500 -16.74400 4.22100 + H -15.93100 -18.16300 3.97500 + O -8.19000 -17.61600 3.01900 + H -8.04100 -17.88800 3.92500 + H -7.35700 -17.23400 2.74300 + O -19.47500 -15.82800 5.33100 + H -18.75800 -16.16000 5.87100 + H -19.19300 -15.99300 4.43100 + O -14.52700 -13.14600 4.57700 + H -13.67000 -13.39700 4.92200 + H -14.86000 -13.94100 4.16200 + O -3.38700 -16.67600 9.80900 + H -4.23300 -16.25000 9.94700 + H -3.01200 -16.75500 10.68600 + O -16.99800 -18.81800 8.91200 + H -17.25500 -18.21100 8.21800 + H -17.70100 -18.75400 9.55900 + O -10.61400 -8.51000 2.15400 + H -11.22200 -8.13500 1.51700 + H -10.70200 -7.95200 2.92700 + O -14.45000 -9.99700 8.04700 + H -14.85900 -9.32400 8.59200 + H -13.57000 -9.66300 7.87400 + O -7.79300 -17.77700 5.73400 + H -8.48100 -17.96800 6.37100 + H -7.18200 -17.20800 6.20200 + O -17.26500 -17.12700 6.68400 + H -16.98500 -16.29400 7.06300 + H -16.58200 -17.34300 6.04900 + O -15.82600 -8.26700 9.72500 + H -15.91900 -7.31600 9.67400 + H -16.64900 -8.56900 10.10900 + O -19.19400 -7.90000 0.83500 + H -19.18900 -7.08900 0.32500 + H -19.15500 -7.61000 1.74600 + O -7.15800 -6.94300 7.84200 + H -8.05300 -6.78800 8.14400 + H -6.83000 -6.07500 7.60600 + O -5.42900 -19.85700 9.07500 + H -5.88200 -19.51500 8.30500 + H -4.51100 -19.62300 8.93700 + O -13.65400 -21.31900 6.61100 + H -13.38900 -21.83900 7.37000 + H -14.56100 -21.07300 6.79300 + O -8.46600 -5.96600 4.61800 + H -8.21700 -6.78400 4.18700 + H -7.64000 -5.49600 4.73600 + O -7.38500 -8.12300 3.40400 + H -6.92700 -8.60000 2.71200 + H -7.06400 -8.51200 4.21700 + O -5.83100 -9.30900 11.12900 + H -5.88700 -9.85000 10.34200 + H -5.65800 -8.42600 10.80200 + O -11.26200 -12.34400 0.91000 + H -12.04900 -11.85300 0.67300 + H -10.65400 -12.18600 0.18800 + O -11.29400 -12.85400 -2.67800 + H -10.74000 -12.30800 -2.12000 + H -11.02700 -13.75100 -2.47800 + O -4.60800 -12.79300 9.67800 + H -5.09800 -12.04800 9.33000 + H -3.86700 -12.89300 9.08000 + O -7.24100 -12.15000 1.58500 + H -7.97600 -11.80600 2.09200 + H -6.67900 -12.57400 2.23300 + O -8.21200 -14.61500 11.19300 + H -8.23000 -13.80600 11.70500 + H -8.77300 -14.43300 10.43900 + O -16.37000 -11.39200 3.38800 + H -16.85100 -11.91900 2.75000 + H -15.68200 -11.97300 3.71000 + O -12.77400 -7.25000 6.14300 + H -12.26000 -6.99700 5.37600 + H -12.28000 -7.97000 6.53600 + O -15.71400 -12.20200 6.88600 + H -15.19200 -12.37800 6.10300 + H -15.26300 -11.47500 7.31600 + O -9.61100 -6.43000 9.04500 + H -9.65400 -5.84700 9.80300 + H -9.99200 -7.25200 9.35300 + O -15.00700 -8.33700 12.40400 + H -15.86400 -8.02800 12.69600 + H -15.11100 -8.48200 11.46300 + O -9.36400 -11.85000 -1.04900 + H -9.03400 -11.04900 -0.64000 + H -8.69800 -12.07900 -1.69800 + O -17.83700 -12.05700 11.64200 + H -17.47900 -11.36000 12.19200 + H -18.78200 -12.01600 11.78700 + O -11.14700 -6.40700 4.01200 + H -10.24400 -6.21200 4.26400 + H -11.40400 -5.67400 3.45300 + O -5.92200 -21.00800 0.66400 + H -5.90500 -21.89400 0.30200 + H -5.41700 -20.48500 0.04200 + O -17.96600 -12.50100 1.41100 + H -18.29600 -11.73100 0.94700 + H -18.72500 -13.07900 1.48700 + O -20.38600 -9.64100 10.44100 + H -21.26100 -9.32900 10.21100 + H -20.54200 -10.35000 11.06600 + O -20.00500 -7.59100 8.12700 + H -20.08700 -8.47500 8.48500 + H -19.50800 -7.10900 8.78800 + O -3.62500 -6.60400 13.42400 + H -3.77500 -7.54900 13.41200 + H -4.40500 -6.24200 13.84600 + O -12.57300 -7.10400 12.62100 + H -12.70600 -6.33300 13.17300 + H -13.43500 -7.51800 12.57600 + O -11.46700 -4.11500 2.33100 + H -11.89800 -4.17300 1.47800 + H -11.68800 -3.24100 2.65300 + O -17.54200 -4.52900 4.35100 + H -17.34600 -4.89600 5.21300 + H -18.08800 -5.19100 3.92600 + O -7.97800 -5.66200 -1.59500 + H -7.45800 -5.94100 -2.34900 + H -7.50900 -6.01200 -0.83800 + O -8.88600 -3.52000 3.20900 + H -9.69900 -3.73100 2.75000 + H -8.57300 -4.36200 3.54200 + O -6.29800 -15.90700 12.85300 + H -6.96700 -15.43800 12.35400 + H -6.22100 -16.75400 12.41400 + O -18.00900 -8.71400 11.41600 + H -17.93200 -9.13300 12.27400 + H -18.80800 -9.08300 11.04000 + O -16.29100 -21.03300 7.32400 + H -17.04800 -21.16500 6.75400 + H -16.55100 -20.32100 7.90900 + O -12.16400 -20.11800 11.43100 + H -11.87700 -20.97700 11.12200 + H -12.43300 -20.26700 12.33800 + O -9.06100 -17.35100 11.68700 + H -9.27400 -17.55100 12.59800 + H -8.89000 -16.40900 11.68200 + O -7.58700 -12.08300 -3.23900 + H -7.02600 -12.09100 -4.01400 + H -7.27400 -11.34100 -2.72300 + O -10.55300 -17.89200 -1.31800 + H -11.39500 -18.24900 -1.60000 + H -9.92300 -18.23200 -1.95200 + O -16.49200 -14.69200 7.76800 + H -17.14900 -14.45300 8.42300 + H -16.22700 -13.85800 7.38000 + O -15.82600 -15.23200 3.25300 + H -16.77600 -15.31800 3.17700 + H -15.50100 -15.36900 2.36300 + O -10.49300 -14.83000 1.73400 + H -10.88600 -13.97900 1.54100 + H -9.68000 -14.83900 1.23100 + O -11.26300 -14.45600 12.76100 + H -11.78900 -15.21900 13.00300 + H -10.43600 -14.57800 13.22700 + O -9.14900 -15.25700 4.12500 + H -9.70500 -14.91500 3.42500 + H -8.82100 -16.08900 3.78500 + O -18.01400 -15.96500 11.20900 + H -17.16200 -16.14700 11.60700 + H -18.41300 -16.82700 11.09400 + O -15.12100 -17.69800 -2.67000 + H -15.53300 -17.82700 -1.81600 + H -15.74400 -17.16100 -3.15900 + O -20.88800 -18.41600 3.92900 + H -20.46500 -18.44500 4.78700 + H -20.37600 -19.01800 3.38900 + O -19.89500 -16.87100 0.46200 + H -19.99400 -16.30400 -0.30300 + H -20.65000 -16.66400 1.01300 + O -9.94000 -14.11000 9.03800 + H -9.85000 -14.59100 8.21500 + H -10.76200 -14.42700 9.41300 + O -12.73200 -18.18000 4.16800 + H -13.60300 -18.00100 4.52100 + H -12.24200 -18.53700 4.90800 + O -18.37200 -16.31800 2.82500 + H -18.87600 -16.29700 2.01200 + H -17.91400 -17.15800 2.80200 + O -14.95800 -15.89800 0.72900 + H -14.03600 -16.03600 0.94800 + H -14.94000 -15.47300 -0.12900 + O -10.06800 -18.15000 7.35500 + H -10.46900 -18.16900 8.22400 + H -10.49500 -18.86200 6.87800 + O -15.37800 -14.75300 -1.75000 + H -15.93600 -15.12500 -2.43300 + H -14.83300 -14.11200 -2.20700 + O -6.20000 -2.79600 14.74300 + H -6.33700 -1.86900 14.54900 + H -5.30400 -2.84200 15.07600 + O -17.51500 -10.20700 13.70800 + H -16.63800 -10.33600 14.07100 + H -18.08600 -10.73000 14.27100 + O -12.35300 -14.00500 6.17600 + H -11.53600 -14.50300 6.19100 + H -12.94500 -14.49700 6.74600 + O -17.05300 -18.71200 2.58100 + H -17.80400 -19.30500 2.60500 + H -16.71500 -18.79000 1.68900 + O -13.81800 -15.46400 8.00200 + H -14.70200 -15.11200 8.10600 + H -13.36300 -15.21000 8.80400 + O -13.48200 -10.90900 0.07300 + H -14.14800 -11.57600 -0.09100 + H -13.91100 -10.27800 0.65100 + O -5.99100 -10.62700 8.73400 + H -5.22500 -10.31100 8.25400 + H -6.62800 -10.84400 8.05400 + O -9.95100 -15.16700 -1.78200 + H -10.20800 -16.06000 -1.55200 + H -9.34800 -14.90500 -1.08600 + O -12.03100 -9.94800 12.07500 + H -11.56000 -10.47000 12.72400 + H -12.18900 -9.11000 12.50900 + O -8.15500 -14.48400 0.28300 + H -7.30000 -14.90600 0.20200 + H -7.95700 -13.59200 0.56800 + O -18.11300 -11.84300 5.56800 + H -17.74800 -11.42300 4.78900 + H -17.36200 -11.97200 6.14600 + O -11.50400 -9.37000 -1.16000 + H -11.78500 -8.57100 -0.71600 + H -12.14800 -10.02900 -0.90100 + O -20.42000 -11.56500 12.52000 + H -20.13700 -11.41500 13.42200 + H -21.01200 -12.31500 12.57600 + O -13.42700 -12.77200 13.39300 + H -14.08000 -13.05200 12.75200 + H -12.61500 -13.19200 13.10700 + O -9.77900 -7.39100 12.82300 + H -10.71900 -7.29800 12.66300 + H -9.68400 -7.27100 13.76800 + O -5.40700 -15.20100 -0.14700 + H -5.61200 -15.55400 -1.01300 + H -5.20400 -14.27900 -0.30600 + O -3.56400 -9.30500 12.71300 + H -4.43600 -9.37800 12.32500 + H -2.99300 -9.78300 12.11300 + O -6.11600 -7.01200 0.08500 + H -6.20100 -7.85100 0.53800 + H -5.63800 -7.22000 -0.71800 + O -7.82900 -11.40600 6.79500 + H -8.78000 -11.48100 6.87700 + H -7.56500 -12.20100 6.33000 + O -15.27300 -7.42000 3.36100 + H -15.14900 -6.47500 3.27100 + H -15.44900 -7.55100 4.29300 + O -20.01500 -12.87000 -0.91100 + H -19.77300 -11.96000 -0.74000 + H -20.29900 -13.20700 -0.06200 + O -15.27900 -9.49000 1.64100 + H -15.20000 -8.70700 2.18600 + H -15.69000 -10.13900 2.21200 + O -13.22600 -5.62500 10.22300 + H -12.62200 -4.89100 10.10700 + H -12.83800 -6.14600 10.92700 + O -15.14800 -4.69000 2.98800 + H -16.00700 -4.49300 3.36000 + H -14.54900 -4.09600 3.43900 + O -10.48600 -8.91700 10.00600 + H -11.14600 -9.24800 10.61500 + H -9.65800 -9.26600 10.33700 + O -8.89900 -9.28700 -0.12100 + H -9.01400 -8.96100 0.77200 + H -9.76600 -9.21100 -0.51800 + O -7.64400 -5.93300 11.94200 + H -8.33800 -6.51700 12.24700 + H -8.10800 -5.18400 11.56700 + O -3.08900 -3.94400 -0.83500 + H -3.20900 -3.54800 -1.69900 + H -3.83800 -3.63800 -0.32500 + O -19.02300 -18.49600 10.76400 + H -19.81400 -18.47300 10.22500 + H -19.31600 -18.84500 11.60500 + O -6.12100 -9.53600 1.35400 + H -6.68400 -10.30800 1.28700 + H -5.23200 -9.89000 1.38900 + O -18.78600 -6.27800 10.27000 + H -18.65400 -7.01000 10.87300 + H -17.92600 -5.86300 10.20400 + O -14.45500 -8.13900 -3.12200 + H -14.52800 -8.96700 -3.59800 + H -14.83800 -8.32200 -2.26400 + O -15.77600 -8.07800 -0.69900 + H -15.71000 -8.64300 0.07100 + H -15.49200 -7.21700 -0.39100 + O -18.94900 -10.40800 -0.24800 + H -19.12900 -9.64000 0.29400 + H -18.60000 -10.04900 -1.06400 + O -11.12500 -17.99200 9.98700 + H -10.35200 -17.82500 10.52600 + H -11.58300 -18.70200 10.43700 + O -9.96400 -18.92000 1.31000 + H -10.11000 -18.43900 0.49500 + H -9.42300 -18.33500 1.84000 + O -12.17900 -7.16900 0.28600 + H -13.05000 -6.80500 0.44300 + H -11.69100 -6.45600 -0.12500 + O -12.50700 -16.72500 1.75100 + H -11.78200 -16.10000 1.74100 + H -12.49300 -17.09500 2.63400 + O -12.13100 -15.33100 10.19200 + H -11.93000 -15.03600 11.08000 + H -11.90200 -16.26100 10.19000 + O -12.04600 -20.29700 2.54900 + H -12.22300 -19.58100 3.16000 + H -11.22600 -20.05000 2.12100 + O -14.58800 -5.75900 0.46200 + H -14.08500 -5.01100 0.14000 + H -14.89600 -5.48600 1.32700 + O -15.89500 -18.36400 0.00000 + H -15.80600 -17.46000 0.30200 + H -15.03200 -18.75200 0.14400 + O -6.39500 -18.20000 11.32600 + H -7.34800 -18.14400 11.26300 + H -6.13400 -18.72100 10.56700 + O -7.20200 -13.90700 5.59900 + H -7.92700 -14.31100 5.12200 + H -6.84200 -14.61600 6.13200 + O -17.70600 -9.11600 -2.51300 + H -17.20200 -8.51800 -1.96100 + H -17.19200 -9.92300 -2.52300 + O -20.76600 -15.34800 -1.84300 + H -20.32200 -14.51800 -1.66600 + H -21.68100 -15.18400 -1.61600 + O -6.06000 -4.39900 7.79700 + H -6.22600 -3.66100 7.21000 + H -5.10700 -4.43000 7.88000 + O -18.50700 -21.61000 5.78500 + H -18.40900 -21.82700 4.85800 + H -18.80700 -22.42300 6.19200 + O -19.40300 -5.80500 2.76400 + H -20.15400 -5.36700 3.16700 + H -19.40400 -5.49800 1.85800 + O -12.11300 -6.75700 -3.28600 + H -12.94100 -7.21700 -3.14900 + H -11.48400 -7.45000 -3.48700 + O -9.38800 -7.11500 15.50600 + H -9.34400 -8.03500 15.76800 + H -8.61800 -6.71400 15.90900 + O -16.36600 -11.47300 -2.46400 + H -16.20700 -11.71500 -1.55200 + H -15.58000 -11.76000 -2.92900 + O -14.82700 -10.50200 14.23500 + H -14.67800 -9.79400 13.60900 + H -14.27200 -11.21800 13.92500 + O -10.31500 -8.79700 -3.60000 + H -10.61200 -9.08800 -2.73800 + H -10.35200 -9.58300 -4.14500 + O -8.46600 -9.64800 11.79100 + H -7.54500 -9.42200 11.66000 + H -8.83700 -8.89100 12.24600 + O -19.27000 -11.91000 15.11100 + H -19.56400 -12.82100 15.11900 + H -19.07300 -11.71400 16.02700 + O -15.66100 -12.59800 -0.05800 + H -16.40300 -12.63600 0.54600 + H -15.64400 -13.46100 -0.47200 + O -15.39400 -13.42800 11.49400 + H -16.27800 -13.06400 11.52500 + H -14.88300 -12.77900 11.01100 + O -13.99700 -12.81200 -3.06600 + H -13.92200 -12.77400 -4.01900 + H -13.09600 -12.73900 -2.75100 + O -10.41500 -11.33000 14.03800 + H -9.69600 -11.66800 13.50400 + H -10.62500 -12.04600 14.63800 + O -6.67800 -10.05300 -1.65900 + H -6.23700 -9.28700 -2.02700 + H -7.32600 -9.69100 -1.05500 + O -15.32700 -16.08000 12.12100 + H -15.06100 -16.57900 11.34900 + H -15.27700 -15.16500 11.84500 + O -14.73100 -17.43500 9.81700 + H -14.23500 -17.01500 9.11400 + H -15.48400 -17.82900 9.37700 + O -13.90600 -11.40900 10.49200 + H -13.23100 -10.91300 10.95400 + H -13.99200 -10.96800 9.64700 + O -9.20200 -15.26500 14.45900 + H -9.34600 -16.21000 14.51900 + H -8.30300 -15.13900 14.76200 + O -7.89700 -12.20300 12.59600 + H -8.14700 -11.35600 12.22600 + H -6.94500 -12.23400 12.50600 + O -16.21600 -15.62000 14.72900 + H -17.16300 -15.53000 14.62500 + H -15.88300 -15.70700 13.83600 + O -14.97000 -20.32400 10.25900 + H -14.33500 -19.63500 10.45400 + H -15.71400 -19.86400 9.87200 + O -12.76300 -16.62500 13.17700 + H -12.66900 -17.42000 13.70300 + H -13.68700 -16.61000 12.92700 + O -9.71000 -18.00100 14.35100 + H -10.61200 -18.28200 14.50200 + H -9.17900 -18.61300 14.86000 + O -8.30500 -19.71300 16.09700 + H -8.40800 -20.62200 16.37800 + H -8.46900 -19.19700 16.88600 + O -19.90300 -17.44300 15.54100 + H -19.83100 -17.98800 14.75800 + H -19.40500 -17.91500 16.20900 + O -13.10000 -21.30700 13.84400 + H -12.87900 -22.06100 14.39000 + H -13.79900 -21.62300 13.27000 + O -12.50900 -18.83000 14.86900 + H -12.64000 -19.73000 14.57000 + H -13.36400 -18.56300 15.20700 + O -12.68800 -3.82800 -0.19000 + H -11.95400 -4.23400 -0.65000 + H -12.97200 -3.11900 -0.76800 + O -19.98600 -8.33600 -3.95900 + H -19.25800 -8.52000 -3.36500 + H -20.17000 -9.17500 -4.38000 + O -21.38700 -4.34100 4.02300 + H -22.31100 -4.46400 3.80500 + H -21.29400 -4.72500 4.89500 + O -20.99500 -5.54600 6.53500 + H -20.70700 -6.36200 6.94400 + H -20.38300 -4.88500 6.86000 + O -3.37700 -4.40300 8.40800 + H -3.27100 -4.81400 9.26600 + H -2.52000 -4.49200 7.99200 + O -3.29200 -5.71200 10.87600 + H -4.07600 -6.14200 10.53300 + H -3.27000 -5.95600 11.80100 + O -4.77400 -8.47200 15.46500 + H -5.59300 -8.65900 15.92400 + H -4.29000 -7.89900 16.06000 + O -3.70200 -10.84500 1.26600 + H -3.06100 -10.14100 1.17000 + H -3.29000 -11.46400 1.86900 + O -2.90700 -12.98400 2.90600 + H -3.81400 -13.22100 3.10100 + H -2.48000 -12.94400 3.76100 + O -3.22700 -10.35100 8.26500 + H -2.89400 -11.19100 7.95100 + H -2.66000 -9.69800 7.85400 + O -5.68400 -6.75900 10.18000 + H -6.37600 -6.41100 10.74200 + H -6.09200 -6.83900 9.31800 + O -4.93700 -12.40000 -0.66700 + H -5.60300 -11.83100 -1.05400 + H -4.61800 -11.91400 0.09300 + O -5.96100 -16.41800 -2.49500 + H -6.09800 -16.01700 -3.35400 + H -6.08500 -17.35500 -2.64600 + O -5.60100 -13.74300 3.32400 + H -5.67100 -14.66600 3.07900 + H -6.05800 -13.68200 4.16200 + O -19.91200 -12.74100 7.53200 + H -19.30700 -12.31200 6.92600 + H -19.35200 -13.07700 8.23200 + O -5.12600 -12.42200 12.37000 + H -4.60700 -13.16700 12.67100 + H -4.94500 -12.36700 11.43200 + O -3.78000 -14.81400 12.61600 + H -4.67100 -15.14400 12.73700 + H -3.22700 -15.59400 12.66700 + O -6.36400 -19.18500 -2.82600 + H -5.57800 -19.37900 -2.31600 + H -6.78300 -20.03700 -2.95500 + O -5.76600 -16.42600 2.37300 + H -5.31700 -17.26900 2.43500 + H -5.55000 -16.10400 1.49800 + O -6.03000 -16.03500 7.04900 + H -5.10100 -16.14300 6.84200 + H -6.04400 -15.80500 7.97800 + O -5.84100 -15.23800 9.70600 + H -6.66000 -15.10700 10.18400 + H -5.41000 -14.38400 9.72800 + O -20.30600 -14.64600 15.23500 + H -20.13000 -15.58700 15.25800 + H -20.71300 -14.50000 14.38100 + O -2.13700 -21.18300 14.73800 + H -2.27300 -22.01500 14.28400 + H -2.69500 -20.56100 14.27000 + O -5.02300 -19.20300 2.62600 + H -4.91300 -19.71700 3.42600 + H -5.43600 -19.80600 2.00700 + O -19.45300 -18.99900 6.27600 + H -19.13600 -19.88700 6.10800 + H -18.65700 -18.47400 6.36400 + O -21.72200 -16.15800 6.95100 + H -20.99400 -16.03200 6.34200 + H -21.35300 -16.68300 7.66100 + O -3.80800 -19.41900 13.47200 + H -4.55500 -19.17500 14.01900 + H -4.18800 -19.94200 12.76600 + O -4.25900 -19.67300 -1.10000 + H -3.43500 -20.14800 -1.20500 + H -3.99700 -18.79900 -0.80900 + O -20.81600 -19.50800 0.14200 + H -21.66700 -19.16100 -0.12600 + H -20.26400 -18.73400 0.24700 + O -6.21200 -4.51100 4.34800 + H -5.90600 -4.83800 3.50200 + H -5.63500 -3.77300 4.54300 + O -20.88200 -18.36900 8.62500 + H -20.51600 -18.73100 7.81800 + H -21.78900 -18.67600 8.63600 + O -16.01200 -5.51500 10.21600 + H -16.19300 -4.57600 10.25300 + H -15.06000 -5.57800 10.29200 + O -12.84200 -4.81600 14.13400 + H -12.56100 -5.12700 14.99400 + H -12.10800 -4.29000 13.81700 + O -10.55600 -3.52900 13.30600 + H -10.02300 -3.76500 12.54600 + H -9.94600 -3.54900 14.04300 + O -4.57600 -21.14200 11.48900 + H -5.01000 -20.76600 10.72300 + H -4.94700 -22.02100 11.56800 + O -19.13700 -2.28700 9.86500 + H -19.22300 -1.33400 9.89000 + H -20.03300 -2.61100 9.95900 + O -16.44800 -2.80300 10.16500 + H -17.38800 -2.64100 10.24100 + H -16.19800 -2.37900 9.34400 + O -11.72100 -3.34200 9.60700 + H -12.09800 -3.02500 8.78600 + H -11.86500 -2.62900 10.22900 + O -9.30300 -4.05400 10.81200 + H -10.03000 -3.74700 10.27000 + H -8.54500 -3.56000 10.49900 + O -6.80100 -3.06400 10.07600 + H -6.54100 -2.18900 9.78800 + H -6.61600 -3.62800 9.32500 + O -18.93200 -3.99600 7.57900 + H -18.83200 -3.67600 8.47500 + H -18.17200 -4.56100 7.43900 + O -16.70600 -5.51400 6.85100 + H -16.55700 -6.43100 6.62000 + H -15.85400 -5.20200 7.15500 + O -14.00300 -5.21300 7.57600 + H -13.55500 -5.83800 7.00600 + H -13.76300 -5.48200 8.46300 + O -11.31900 -20.08000 5.74800 + H -12.06000 -20.58900 6.07500 + H -10.79400 -20.71000 5.25400 + O -5.54200 20.96600 6.03100 + H -5.01500 21.57000 5.50800 + H -6.39000 21.40200 6.11900 + O -4.01100 -21.15900 4.51600 + H -3.25000 -21.62000 4.16200 + H -3.64500 -20.39100 4.95400 + O -19.35700 -20.21400 2.46100 + H -19.87700 -20.26600 1.65900 + H -19.08900 -21.11700 2.63300 + O -19.81700 -21.57100 -1.44700 + H -18.93600 -21.19900 -1.42800 + H -20.33500 -20.99200 -0.88800 + O -14.41900 -21.25300 3.93900 + H -13.65800 -20.91600 3.46600 + H -14.15500 -21.24100 4.85900 + O -9.64800 21.96600 4.25500 + H -9.27800 22.35100 3.46000 + H -9.37200 21.05000 4.22800 + O -8.46800 -21.15700 1.83400 + H -8.96800 -20.37800 1.59300 + H -7.57900 -20.97700 1.52700 + O -19.58700 -5.11900 0.01000 + H -20.48400 -5.09800 -0.32500 + H -19.37800 -4.20300 0.19300 + O -17.52000 -20.14600 -1.38200 + H -17.16400 -20.08300 -2.26900 + H -17.00400 -19.52200 -0.87200 + O -13.28800 -19.02300 0.43100 + H -12.95400 -19.67500 1.04700 + H -12.95500 -18.18800 0.76200 + O -10.74000 -5.38600 -1.30900 + H -9.79000 -5.48000 -1.39000 + H -11.08900 -5.79200 -2.10200 + O -5.13200 -5.12200 1.88400 + H -5.49600 -5.81300 1.33000 + H -5.16900 -4.33400 1.34100 + O -19.34600 -19.36400 13.52600 + H -19.74800 -20.17600 13.83400 + H -18.44300 -19.40800 13.84200 + O -16.89300 -19.56900 14.81300 + H -16.08100 -19.12000 15.05000 + H -16.64300 -20.49000 14.73500 + O -18.22400 -4.44700 -2.49200 + H -18.48000 -4.77700 -1.63100 + H -17.56100 -5.06600 -2.80000 + O -8.41200 21.67100 17.20600 + H -9.07600 21.00400 17.37700 + H -7.64700 21.17800 16.91000 + O -6.05000 -5.49600 14.18100 + H -6.56400 -5.70100 13.40000 + H -6.12100 -4.54600 14.27200 + O 9.32000 -11.49100 2.73300 + H 9.13600 -10.55500 2.64800 + H 8.59700 -11.92000 2.27500 + O 9.27600 -12.21400 7.81600 + H 9.06800 -12.76700 8.57000 + H 8.88000 -12.66500 7.07100 + O -0.61100 -14.55300 9.28500 + H -0.57500 -15.43700 9.65100 + H 0.26900 -14.39900 8.94100 + O 12.03900 -9.35200 5.39400 + H 12.03700 -10.16200 5.90400 + H 12.16900 -8.66000 6.04200 + O 1.22700 -9.17200 6.54900 + H 1.11200 -10.05500 6.19700 + H 2.17600 -9.05000 6.58500 + O -1.03200 -14.61400 1.55700 + H -1.77600 -14.11100 1.88700 + H -1.33000 -14.96000 0.71600 + O -1.38000 -13.00400 5.33000 + H -1.09100 -13.89300 5.53300 + H -0.57700 -12.48300 5.33500 + O 9.24000 -4.76400 5.43700 + H 9.70200 -5.02500 4.64000 + H 9.22200 -5.55600 5.97400 + O 5.87600 -10.01600 6.93500 + H 5.93800 -10.93300 6.66800 + H 6.77200 -9.77200 7.16600 + O 16.30800 -19.22800 9.21200 + H 16.32700 -18.47800 9.80800 + H 16.56500 -18.86700 8.36400 + O 8.10000 -15.73500 7.21300 + H 8.35400 -16.64700 7.35300 + H 8.49000 -15.50500 6.36900 + O 3.12600 -17.30700 4.98000 + H 3.14300 -16.40900 4.64700 + H 2.55300 -17.77700 4.37300 + O 10.72900 -17.46000 3.52000 + H 11.08500 -17.52400 4.40600 + H 11.39400 -16.97700 3.02900 + O -0.65700 -15.55900 6.10800 + H 0.02800 -15.91300 6.67600 + H -0.36800 -15.77400 5.22100 + O 3.24200 -13.43800 5.98000 + H 3.97700 -13.78100 6.48900 + H 3.23000 -13.97300 5.18600 + O 16.34800 -17.19800 11.03300 + H 15.71600 -16.60000 10.63500 + H 16.11200 -17.21200 11.96100 + O 2.51800 -19.10300 9.40500 + H 2.66300 -18.16700 9.54600 + H 1.70000 -19.29100 9.86400 + O 8.84500 -8.89400 1.90100 + H 8.50600 -9.16900 1.05000 + H 8.44900 -8.03600 2.05200 + O 5.10800 -9.47300 9.58500 + H 4.19700 -9.19500 9.67900 + H 5.20300 -9.67000 8.65300 + O 11.30500 -17.40000 6.16900 + H 10.59700 -17.74100 6.71500 + H 11.93900 -17.04500 6.79200 + O 1.50100 -16.79100 7.40400 + H 1.93600 -16.61500 8.23800 + H 2.21700 -16.95100 6.78900 + O 2.55800 -8.50000 10.16400 + H 2.46700 -7.54900 10.21800 + H 1.69300 -8.83800 10.39800 + O -2.03100 -8.71700 0.52200 + H -2.16100 -8.35800 -0.35600 + H -1.70600 -7.97800 1.03700 + O 12.08800 -7.22800 7.21700 + H 11.14100 -7.13700 7.11200 + H 12.43000 -6.34300 7.08600 + O 13.76200 -20.24900 9.30000 + H 13.63200 -20.54100 8.39700 + H 14.64400 -19.87700 9.30700 + O 5.15600 -21.55800 6.52800 + H 5.40300 -22.22700 7.16600 + H 4.28600 -21.27200 6.80500 + O 10.54800 -5.40300 3.04300 + H 10.86800 -6.30500 3.07700 + H 11.33500 -4.86600 3.13000 + O 11.29300 -8.03300 3.08000 + H 10.51900 -8.48100 2.73900 + H 11.54900 -8.54600 3.84700 + O 12.51000 -9.58800 11.39000 + H 12.78900 -10.27900 10.78900 + H 12.70700 -8.77300 10.92700 + O 7.57500 -12.69100 0.90700 + H 6.75800 -12.28500 0.61900 + H 8.16800 -12.59000 0.16200 + O 8.38500 -13.41100 -2.93100 + H 8.90600 -13.30700 -2.13500 + H 8.68600 -14.23800 -3.30800 + O 15.43800 -12.98900 9.95000 + H 14.71600 -12.36000 9.97500 + H 15.89000 -12.80200 9.12700 + O 11.66500 -12.04900 1.46500 + H 10.91300 -11.83000 2.01500 + H 12.25400 -12.53200 2.04500 + O 10.19000 -14.97900 11.54800 + H 10.26900 -14.23100 12.14100 + H 9.62000 -14.66800 10.84500 + O 2.38500 -12.04400 2.89000 + H 1.83100 -12.24700 2.13600 + H 2.72100 -12.89300 3.17700 + O 3.85900 -8.32600 6.18100 + H 3.88300 -8.24300 5.22800 + H 4.59000 -8.90800 6.39000 + O 9.34700 -7.02400 7.11500 + H 9.07900 -6.48500 7.85900 + H 8.97200 -7.88600 7.29600 + O 3.92700 -8.61700 12.59400 + H 3.23800 -8.07300 12.97600 + H 3.61200 -8.81800 11.71300 + O 10.20300 -13.10800 -0.69900 + H 10.69000 -12.74000 0.03900 + H 10.69900 -12.83900 -1.47200 + O 0.16100 -13.38400 13.70700 + H 0.55100 -12.51100 13.66000 + H -0.39400 -13.44100 12.92900 + O 8.01300 -6.27900 2.26600 + H 8.81600 -5.94300 2.66400 + H 7.53800 -5.49700 1.98500 + O 1.85100 -6.85000 13.21200 + H 2.03500 -6.40300 12.38700 + H 0.90500 -6.76200 13.32700 + O 14.00100 -21.47300 0.01200 + H 13.62800 -20.86600 0.65200 + H 14.25300 -20.91900 -0.72600 + O 0.82700 -12.72300 0.74900 + H 0.21400 -12.11600 0.33500 + H 0.28300 -13.45900 1.02900 + O 16.12300 -7.35400 -1.60100 + H 15.90600 -6.42900 -1.71700 + H 16.85600 -7.50500 -2.19800 + O -2.11100 -10.68300 10.85300 + H -2.55200 -10.67100 10.00400 + H -1.97700 -11.61200 11.04100 + O -1.34000 -8.52800 7.39900 + H -0.46500 -8.79300 7.11800 + H -1.19400 -8.03600 8.20800 + O 6.24000 -7.19200 12.05600 + H 6.03500 -6.42300 12.58800 + H 5.53100 -7.80800 12.24200 + O 7.15700 -3.71900 1.40100 + H 6.54600 -3.70400 0.66400 + H 6.79500 -3.09100 2.02600 + O 1.09200 -6.13400 3.53000 + H 0.91400 -6.29700 4.45700 + H 0.33000 -6.49200 3.07400 + O 10.31600 -4.22300 -2.12200 + H 10.84800 -4.99900 -2.29900 + H 10.29400 -4.16100 -1.16700 + O 9.89000 -4.01800 0.66800 + H 8.95600 -3.85900 0.80600 + H 10.15100 -4.56900 1.40600 + O 12.68900 -15.97600 12.22500 + H 11.82500 -15.64400 11.98300 + H 12.68100 -16.89200 11.94700 + O 0.11700 -9.15600 11.33000 + H 0.47500 -9.56400 12.11800 + H -0.64700 -9.68900 11.10900 + O 2.61000 -20.89100 7.25800 + H 1.92700 -20.60800 6.65000 + H 2.56900 -20.26000 7.97600 + O 6.85900 -20.69100 11.35400 + H 7.02500 -21.61900 11.18800 + H 6.37400 -20.67800 12.17900 + O 9.97700 -17.72200 11.74000 + H 9.66800 -17.85600 12.63500 + H 9.95900 -16.77200 11.61900 + O 11.26700 -12.12300 -3.03100 + H 11.13800 -12.29500 -3.96400 + H 12.19000 -11.87700 -2.95800 + O 7.10200 -18.50700 -1.36500 + H 6.37300 -18.64100 -0.75900 + H 6.69600 -18.50600 -2.23200 + O 2.78500 -14.80300 3.57100 + H 1.86900 -15.07900 3.61300 + H 3.08600 -15.10500 2.71400 + O 8.17100 -14.97300 2.30400 + H 7.89100 -14.14800 1.90900 + H 8.81400 -15.32700 1.69000 + O 7.03000 -14.47300 13.48600 + H 6.98100 -15.09300 14.21400 + H 6.98500 -15.02100 12.70200 + O 9.38100 -15.28900 4.78600 + H 8.88300 -14.94800 4.04300 + H 9.73000 -16.12500 4.47700 + O 0.03200 -16.92500 10.44300 + H 0.96700 -16.76100 10.32000 + H -0.03100 -17.87200 10.56800 + O 3.94800 -18.23500 -2.31800 + H 3.57000 -18.41000 -1.45600 + H 3.43800 -17.50000 -2.65900 + O -2.97800 -18.87200 5.62600 + H -2.11100 -18.89600 6.03300 + H -3.34300 -18.02700 5.88900 + O -3.40900 -17.21600 -0.11100 + H -2.67100 -16.71700 -0.46200 + H -4.13800 -16.59500 -0.10700 + O 8.36300 -14.26100 9.52900 + H 8.25200 -14.79900 8.74500 + H 7.51800 -14.31300 9.97700 + O 5.77800 -18.17100 4.57100 + H 4.90700 -17.93500 4.89100 + H 6.22400 -18.53500 5.33600 + O 0.29700 -15.93200 3.56400 + H -0.20600 -15.55600 2.84200 + H 0.54500 -16.80400 3.25600 + O 3.64400 -15.77000 1.12800 + H 4.52700 -15.97700 1.43300 + H 3.77500 -15.34900 0.27800 + O 9.00800 -18.35800 7.49600 + H 9.03900 -18.79600 8.34600 + H 8.48100 -18.93900 6.94600 + O 3.94400 -14.77700 -1.41900 + H 3.47100 -15.25100 -2.10300 + H 4.63800 -14.31100 -1.88600 + O 13.22600 -3.84300 13.55600 + H 13.79900 -3.84500 12.79000 + H 13.43000 -3.02300 14.00700 + O 0.72500 -10.55800 13.78600 + H 1.34700 -10.60100 14.51200 + H -0.04600 -10.12700 14.15500 + O 5.41700 -14.64300 7.21400 + H 6.27000 -15.07600 7.17100 + H 5.03500 -14.94700 8.03800 + O 1.47600 -18.34600 2.88900 + H 0.89700 -19.10300 2.97500 + H 1.86300 -18.43700 2.01900 + O 5.33300 -11.26600 -0.00900 + H 4.65000 -11.65100 -0.55800 + H 4.85600 -10.75700 0.64700 + O 13.46400 -11.11900 9.28400 + H 14.16300 -10.55300 8.95700 + H 12.92400 -11.30300 8.51500 + O 9.48400 -18.44200 -0.08600 + H 8.64000 -18.42900 -0.53800 + H 9.70000 -17.51800 0.04400 + O 7.63000 -9.26500 10.68500 + H 6.78600 -9.56100 10.34600 + H 7.42000 -8.49900 11.21900 + O 10.10600 -15.72800 0.47000 + H 10.96900 -15.79900 0.87700 + H 10.15300 -14.92800 -0.05300 + O 1.17200 -11.72800 5.39400 + H 1.48200 -11.64100 4.49300 + H 1.82300 -12.28600 5.81900 + O 7.64300 -9.83500 -0.54100 + H 7.32100 -8.93900 -0.63900 + H 6.85600 -10.35900 -0.39400 + O -1.63900 -13.34400 11.56600 + H -2.41000 -13.83700 11.84500 + H -1.33300 -13.80000 10.78100 + O 4.62400 -13.17300 13.62200 + H 4.12800 -13.56100 12.90200 + H 5.51400 -13.50500 13.50700 + O 8.80700 -6.21900 12.48400 + H 7.97100 -6.59700 12.21200 + H 9.09600 -6.77200 13.21000 + O 13.07100 -15.61700 -0.89000 + H 12.65600 -15.73300 -1.74500 + H 13.35800 -14.70300 -0.88500 + O 14.96400 -10.14400 12.88000 + H 14.17800 -9.62500 12.70800 + H 15.38900 -9.69900 13.61300 + O 13.32700 -7.25600 1.35500 + H 12.66700 -7.40700 2.03300 + H 13.33800 -8.06700 0.84700 + O 11.90900 -11.70500 7.01600 + H 11.01200 -11.78100 7.33900 + H 12.02900 -12.47300 6.45800 + O 3.50300 -7.75400 3.48400 + H 4.03200 -7.02200 3.16500 + H 2.60700 -7.41700 3.49400 + O -2.38800 -12.96300 -1.61100 + H -1.85700 -12.25000 -1.25600 + H -3.27800 -12.77000 -1.31400 + O 3.68900 -9.88200 1.72800 + H 3.72100 -9.18200 2.38000 + H 3.24500 -10.60400 2.17200 + O 5.13300 -5.90000 9.77500 + H 5.73300 -5.17800 9.58800 + H 5.63700 -6.49300 10.33200 + O 3.50900 -5.11200 2.54300 + H 2.61700 -5.26200 2.85500 + H 3.85600 -4.43400 3.12300 + O 8.40500 -9.53900 8.01300 + H 8.26200 -9.40200 8.94900 + H 8.85200 -10.38300 7.95900 + O 12.21400 -8.72600 -2.02100 + H 12.51800 -9.05600 -1.17600 + H 11.34800 -9.11700 -2.13400 + O 11.17300 -5.95000 11.14600 + H 10.27100 -6.05900 11.44700 + H 11.14300 -5.19000 10.56500 + O 15.64900 -4.60300 -1.97500 + H 15.65300 -4.01600 -2.73100 + H 15.05300 -4.18600 -1.35200 + O 0.20400 -19.70400 10.78600 + H -0.49300 -20.29000 10.49100 + H 0.31500 -19.91000 11.71400 + O 13.02000 -9.87600 0.31400 + H 12.42900 -10.51000 0.72200 + H 13.89400 -10.24200 0.44800 + O -0.99400 -7.01600 9.74400 + H -0.55600 -7.60500 10.35900 + H -1.71500 -6.63800 10.24700 + O 4.74700 -8.57200 -3.05500 + H 4.81400 -9.47400 -3.37000 + H 4.35700 -8.65000 -2.18500 + O 3.30100 -8.45700 -0.66400 + H 3.40500 -8.97200 0.13500 + H 3.57300 -7.57300 -0.41700 + O -0.79200 -10.92800 -0.55400 + H -1.27100 -10.26000 -0.06300 + H -0.17300 -10.43300 -1.09100 + O 8.97400 -19.57000 9.99500 + H 9.32500 -18.92400 10.60800 + H 8.19500 -19.91600 10.43100 + O 9.54200 -19.64800 2.39100 + H 9.48700 -19.36200 1.47900 + H 9.85700 -18.87800 2.86500 + O 6.66900 -7.32000 -0.65200 + H 5.85100 -6.98200 -0.28800 + H 7.14100 -6.54100 -0.94600 + O 5.99700 -16.64500 2.28200 + H 6.74500 -16.05000 2.32900 + H 5.95100 -17.04300 3.15100 + O 6.54900 -16.08900 11.18900 + H 5.75700 -15.81100 10.72900 + H 6.97700 -16.69600 10.58500 + O 6.69900 -20.14300 2.84800 + H 6.46300 -19.43600 3.44900 + H 7.63200 -20.01300 2.68000 + O 4.33400 -5.98500 0.09900 + H 4.58700 -5.17500 -0.34200 + H 4.03000 -5.70300 0.96200 + O 2.73700 -18.27300 0.33500 + H 2.85100 -17.34800 0.55300 + H 3.59000 -18.66900 0.51100 + O 12.62000 -18.60800 11.35100 + H 11.66300 -18.57100 11.33900 + H 12.84800 -19.14800 10.59500 + O 11.88300 -14.18900 5.60900 + H 11.01100 -14.52200 5.40000 + H 12.26500 -14.85700 6.17900 + O 0.85200 -9.14400 -1.91700 + H 1.57300 -8.72800 -1.44300 + H 1.28100 -9.66700 -2.59400 + O -1.52200 -15.49400 -1.04200 + H -1.77500 -14.63800 -1.38700 + H -1.00000 -15.89000 -1.74100 + O 13.07000 -4.64000 7.30400 + H 12.93400 -3.80300 6.86100 + H 13.98800 -4.62100 7.57600 + O 0.42800 -20.69800 5.53500 + H 0.17000 -20.56200 4.62400 + H 0.46900 -21.64900 5.63400 + O -1.12600 -6.36400 1.79500 + H -1.82200 -5.90200 2.26200 + H -0.81200 -5.73400 1.14700 + O -1.23500 -4.18800 14.99000 + H -0.62600 -4.33700 15.71200 + H -1.18200 -4.98500 14.46200 + O 2.71500 -6.49600 15.82400 + H 3.56500 -6.92600 15.90700 + H 2.47500 -6.61200 14.90500 + O 6.71800 -6.77000 -3.91500 + H 6.05100 -7.35000 -3.54700 + H 7.52900 -7.27600 -3.86800 + O 9.82900 -7.72600 14.55500 + H 9.99400 -8.62100 14.85200 + H 10.70100 -7.35500 14.41700 + O 11.63300 -6.35700 -3.23500 + H 11.98500 -7.10500 -2.75200 + H 12.11100 -6.36100 -4.06400 + O -1.67300 -9.31400 14.75500 + H -1.77000 -10.23600 14.99300 + H -2.39600 -9.14500 14.15100 + O 2.23700 -10.94600 -3.50100 + H 2.34300 -11.43900 -2.68800 + H 3.08000 -11.03100 -3.94600 + O 4.69200 -10.47200 14.48600 + H 4.53800 -9.91100 13.72500 + H 4.78200 -11.35200 14.12000 + O 9.74000 -9.88000 -2.33200 + H 9.04500 -9.98100 -1.68200 + H 10.04800 -10.77100 -2.49500 + O 9.91000 -10.54100 11.94200 + H 10.68300 -10.02200 11.71900 + H 9.18600 -10.09400 11.50300 + O -1.80100 -11.96000 15.13700 + H -1.32900 -12.59800 14.60300 + H -1.78800 -12.32900 16.02000 + O 2.84700 -12.26600 -1.11200 + H 2.17500 -12.35100 -0.43600 + H 3.18800 -13.15300 -1.22400 + O 5.63400 -13.10300 -2.74500 + H 5.43800 -12.30600 -3.23800 + H 6.58700 -13.18300 -2.78800 + O 8.71500 -12.67900 14.96400 + H 9.46500 -12.82800 14.38800 + H 7.99000 -13.13600 14.53700 + O 13.96400 -11.49100 -3.13700 + H 14.06900 -11.88600 -4.00300 + H 14.29500 -10.59900 -3.23900 + O 9.78400 -15.97900 15.89800 + H 9.49600 -16.66600 15.29600 + H 10.68800 -15.79900 15.64200 + O 10.79500 -12.79300 13.15800 + H 10.46500 -12.01400 12.71100 + H 11.74100 -12.65500 13.21500 + O 1.66900 -15.78300 13.51300 + H 0.97500 -16.43700 13.59000 + H 1.22000 -14.94400 13.61100 + O 4.34100 -21.00000 10.09900 + H 5.11400 -20.55900 10.45100 + H 3.75100 -20.28700 9.85100 + O 6.34200 -17.70100 13.50900 + H 5.44400 -17.65000 13.83500 + H 6.32900 -17.20800 12.68900 + O 9.00100 -18.04700 14.28000 + H 8.05600 -18.11900 14.14400 + H 9.23600 -18.84200 14.75800 + O 10.37900 -19.93100 16.00000 + H 10.43200 -20.86900 16.18300 + H 9.94600 -19.56100 16.77000 + O -0.22300 -17.70300 14.05900 + H -0.06200 -18.55100 13.64500 + H -0.07600 -17.85900 14.99200 + O 5.11700 22.09100 13.27600 + H 5.63200 21.30800 13.47100 + H 4.37100 21.76700 12.77000 + O 4.84700 -20.06700 15.33300 + H 4.96500 -20.66800 14.59700 + H 4.35300 -19.33300 14.96700 + O 5.59500 -3.68700 -0.89800 + H 6.32700 -4.18200 -1.26700 + H 5.42400 -2.99700 -1.53900 + O -1.82100 -8.40600 -2.39400 + H -0.89200 -8.47100 -2.17300 + H -1.95000 -9.05700 -3.08400 + O -2.88100 -4.86700 3.46800 + H -3.67700 -5.06800 2.97600 + H -2.85900 -5.52200 4.16600 + O -2.22500 -6.51700 5.55100 + H -2.19300 -7.40600 5.90600 + H -1.88500 -5.96500 6.25500 + O 15.64500 -4.50800 8.35800 + H 15.60100 -5.15800 9.06000 + H 16.57100 -4.47800 8.11700 + O 15.38500 -6.19700 10.51200 + H 14.55500 -6.60900 10.27400 + H 15.91900 -6.91300 10.85700 + O 15.46200 -11.07900 0.71300 + H 16.02200 -10.31100 0.83000 + H 15.66000 -11.63600 1.46500 + O 15.90800 -12.93700 2.70200 + H 15.02700 -13.17500 2.99100 + H 16.37800 -12.73000 3.51000 + O 15.82300 -9.92400 8.49300 + H 16.14300 -10.73000 8.08800 + H 16.23900 -9.22200 7.99200 + O 13.03700 -7.39800 9.80700 + H 12.33400 -6.89600 10.22000 + H 12.77500 -7.46600 8.88900 + O 13.93900 -13.03600 -0.86200 + H 13.98400 -12.52300 -1.66900 + H 14.22100 -12.43000 -0.17700 + O 12.17500 -16.17000 -3.49300 + H 12.41400 -15.85800 -4.36600 + H 12.33600 -17.11300 -3.52300 + O 13.21400 -13.69900 3.16700 + H 13.02400 -14.54300 2.75900 + H 12.82500 -13.76000 4.03900 + O -2.45100 -12.97500 7.92400 + H -2.22400 -13.02600 6.99600 + H -1.83300 -13.56500 8.35600 + O 13.50900 -12.43100 13.63300 + H 14.06900 -13.20200 13.54300 + H 14.02700 -11.71400 13.26700 + O 14.91300 -14.82600 13.33200 + H 14.11000 -15.13700 12.91500 + H 15.31200 -15.61400 13.70200 + O 12.61000 -18.90400 -3.62900 + H 13.25600 -19.30600 -3.04800 + H 11.76800 -19.23300 -3.31500 + O 12.65100 -16.13500 1.80600 + H 13.22200 -16.84100 2.11000 + H 12.90800 -15.99400 0.89500 + O 13.47100 -16.03300 7.32400 + H 14.20700 -15.59100 6.90200 + H 13.66800 -15.98800 8.26000 + O 14.37700 -15.53600 9.96500 + H 13.73500 -15.49700 10.67400 + H 14.76400 -14.66100 9.94400 + O -2.50500 -17.19300 12.48800 + H -1.64100 -17.23300 12.89800 + H -2.95900 -17.97200 12.80800 + O 15.85300 -21.75400 14.67100 + H 16.13600 -22.48800 14.12600 + H 15.52200 -21.10700 14.04800 + O 13.98000 -18.30600 2.88500 + H 14.17100 -18.26700 3.82200 + H 13.42500 -19.08000 2.78500 + O -0.60400 -18.71500 7.12700 + H -0.16600 -19.39500 6.61600 + H 0.08300 -18.07500 7.31400 + O -3.24300 -16.36700 6.99000 + H -2.39900 -16.03900 6.67900 + H -3.15600 -16.38900 7.94300 + O 14.51400 -19.77900 13.03900 + H 13.80300 -19.25900 12.66600 + H 14.55800 -20.55800 12.48400 + O 14.79500 -19.80900 -2.09500 + H 15.43000 -20.04300 -2.77200 + H 15.04900 -18.92700 -1.82200 + O -2.44000 -19.20700 1.59600 + H -3.24700 -19.24500 2.10800 + H -2.56100 -18.46000 1.01000 + O 12.84300 -3.94800 3.11600 + H 13.60800 -4.38900 2.74600 + H 13.20200 -3.18900 3.57600 + O -2.05400 -21.30600 10.31100 + H -2.81300 -21.31400 10.89400 + H -2.26700 -20.64900 9.64900 + O -0.92300 -6.73300 13.79900 + H -1.84600 -6.62100 13.57200 + H -0.88300 -7.59000 14.22400 + O 2.57700 -5.76800 10.76700 + H 2.29700 -4.86300 10.62600 + H 3.48200 -5.78900 10.45400 + O 6.09000 -4.97700 13.71500 + H 6.42400 -5.53400 14.41800 + H 6.80600 -4.37000 13.52900 + O 8.41500 -3.57200 13.15200 + H 8.65900 -4.42100 12.78400 + H 9.12300 -3.36100 13.76000 + O 14.55900 21.99000 11.38800 + H 14.12700 22.36300 10.61900 + H 14.04700 21.20800 11.59500 + O 0.15400 -1.52300 11.53500 + H -0.69900 -1.49500 11.10100 + H -0.05300 -1.58800 12.46700 + O 2.09900 -3.08700 10.32300 + H 1.36800 -2.61600 10.72400 + H 2.09900 -2.79600 9.41100 + O 6.57000 -3.64300 9.12400 + H 6.01000 -3.24900 8.45500 + H 6.68900 -2.95200 9.77500 + O 8.82900 -5.22300 9.09400 + H 8.05900 -4.65500 9.06400 + H 9.57000 -4.61900 9.14100 + O 11.30800 -3.90500 9.32300 + H 11.81000 -3.12000 9.54200 + H 11.76300 -4.27800 8.56800 + O -0.80000 -5.19400 7.63100 + H -0.80600 -5.71300 8.43500 + H -0.05900 -5.53600 7.13000 + O 1.24300 -6.39000 6.30700 + H 1.25000 -7.33900 6.42800 + H 2.09600 -6.10000 6.63200 + O 3.81900 -5.81100 7.34300 + H 3.96900 -6.65700 6.92100 + H 4.22200 -5.89900 8.20700 + O 7.40700 -20.05600 5.97300 + H 6.65700 -20.58500 6.24400 + H 7.90400 -20.62600 5.38600 + O 12.63600 20.99500 6.69900 + H 13.28100 21.68900 6.56200 + H 11.79000 21.44200 6.65800 + O 14.35400 -20.75300 6.53200 + H 15.28000 -20.95100 6.39300 + H 14.23700 -19.87900 6.16000 + O -0.33300 -20.43200 2.89800 + H -1.07200 -19.98900 2.48000 + H -0.32900 -21.30700 2.51100 + O -1.66500 -20.66100 -0.72500 + H -0.90900 -20.11400 -0.93700 + H -1.85100 -20.46900 0.19400 + O 4.40000 -21.44700 3.87300 + H 5.12300 -20.96400 3.47200 + H 4.60500 -21.45300 4.80800 + O 9.00500 -21.66300 4.24700 + H 9.25300 -21.00500 3.59600 + H 9.47400 -22.45300 3.98000 + O 12.35900 -20.47500 2.23600 + H 11.43300 -20.23300 2.25700 + H 12.36100 -21.42500 2.35200 + O -0.53600 -4.28900 0.05900 + H -1.42500 -4.26600 -0.29600 + H -0.30300 -3.36900 0.18400 + O 0.60100 -19.12600 -1.23000 + H 1.00100 -19.71700 -1.86700 + H 1.31200 -18.90600 -0.62700 + O 5.35500 -18.84100 0.77900 + H 5.72900 -19.53800 1.31800 + H 5.60700 -18.03200 1.22500 + O 7.66400 -5.05800 -1.95700 + H 8.54300 -4.72400 -2.13400 + H 7.42000 -5.52700 -2.75500 + O 15.01900 -5.10600 1.54300 + H 14.47400 -5.88300 1.41700 + H 14.72900 -4.49700 0.86500 + O 0.18100 -20.34000 13.43400 + H -0.61700 -20.67400 13.84200 + H 0.89000 -20.67800 13.98200 + O 1.77400 -21.39800 15.41300 + H 2.45100 -21.59700 16.06100 + H 1.68100 -22.20600 14.90900 + O 1.10300 -4.63700 -2.29500 + H 0.77400 -4.76000 -1.40500 + H 1.64900 -5.40700 -2.45700 + O 10.64200 21.33300 16.74400 + H 10.05500 20.57900 16.79900 + H 11.51600 20.95100 16.65900 + O 12.20200 -6.43200 13.68200 + H 11.89800 -6.44600 12.77500 + H 12.68400 -5.60900 13.76100 + O 18.78800 -13.78900 8.20700 + H 18.84500 -14.32100 9.00000 + H 19.69700 -13.65200 7.94300 + O 20.05500 -8.81600 7.14100 + H 20.13900 -9.68800 6.75500 + H 20.58000 -8.85600 7.94000 + O 17.68400 -14.69400 1.41700 + H 17.00800 -14.15400 1.82700 + H 17.33500 -14.89500 0.54800 + O 17.66000 -12.52700 4.89500 + H 17.94000 -13.44300 4.89400 + H 18.45900 -12.03300 5.07800 + O 20.47600 -19.13500 4.65800 + H 21.36900 -18.93900 4.37700 + H 19.98100 -19.22100 3.84300 + O 18.18700 -15.24000 4.99900 + H 18.63900 -15.72900 5.68600 + H 18.55000 -15.58100 4.18200 + O 20.28500 -18.83900 8.54000 + H 19.95700 -18.17800 7.93000 + H 19.88900 -18.60800 9.38000 + O 19.68500 -17.16400 6.38600 + H 20.49300 -16.68400 6.57200 + H 19.93600 -17.80900 5.72500 + O 21.21000 -8.27000 9.71600 + H 21.63300 -7.41100 9.71900 + H 20.49100 -8.18600 10.34100 + O 17.09400 -8.77400 0.61100 + H 16.67900 -8.33300 -0.13000 + H 17.39300 -8.06300 1.17700 + O 21.06000 -12.53500 3.72100 + H 20.87700 -12.74800 2.80600 + H 21.97400 -12.78900 3.84800 + O 21.44800 -13.54800 7.53900 + H 21.62100 -14.44800 7.26300 + H 22.30500 -13.12200 7.51700 + O 19.42300 -13.64900 13.86500 + H 19.13200 -13.03700 14.54100 + H 18.69800 -13.68000 13.24200 + O 19.98800 -13.12700 1.06600 + H 19.69500 -12.26900 0.75900 + H 19.18500 -13.57400 1.33300 + O 16.51900 -8.77900 10.95300 + H 16.24400 -9.22000 10.14800 + H 16.11300 -9.28700 11.65500 + O 17.47000 -7.93900 7.48900 + H 18.32300 -8.29800 7.24400 + H 17.64000 -7.43900 8.28700 + O 19.74300 -5.08800 3.70100 + H 19.74400 -5.59900 4.51000 + H 19.18600 -5.58500 3.10300 + O 19.06300 -7.85300 11.49700 + H 19.21800 -8.09500 12.41000 + H 18.19200 -8.20000 11.30200 + O 19.77000 -20.91000 6.70800 + H 20.10600 -20.40400 5.96800 + H 19.96600 -20.37300 7.47500 + O 21.67200 -14.11800 10.39100 + H 20.79900 -14.51000 10.41700 + H 21.76800 -13.80400 9.49200 + O 22.08000 -16.29600 2.30800 + H 21.13400 -16.26000 2.45000 + H 22.39400 -16.93400 2.95000 + O 19.15100 -15.18500 10.53400 + H 18.48900 -14.85200 11.14000 + H 19.15200 -16.13100 10.67700 + O 14.49300 -18.11200 5.62600 + H 15.38700 -18.00400 5.95100 + H 13.98300 -17.45900 6.10600 + O 15.29200 -17.22900 -1.14900 + H 15.95500 -16.54500 -1.24400 + H 14.46200 -16.75300 -1.09800 + O 19.31300 -16.52000 2.72800 + H 18.73400 -15.98900 2.18100 + H 19.04600 -17.42300 2.55400 + O 20.58800 -14.94400 -0.96200 + H 19.88500 -14.73000 -1.57600 + H 20.47200 -14.32700 -0.23900 + O 18.91100 -19.34000 2.38400 + H 18.32600 -19.97600 2.79600 + H 18.91100 -19.57900 1.45700 + O 20.15300 -11.44800 6.05500 + H 20.50600 -11.67600 5.19500 + H 20.53500 -12.09400 6.65000 + O 17.19400 -13.87300 12.06200 + H 16.46800 -14.17100 12.61000 + H 16.77500 -13.39600 11.34600 + O 17.01000 -12.32100 -1.22200 + H 17.79600 -11.85900 -0.92900 + H 16.29900 -11.93300 -0.71200 + O 18.99000 -17.94900 10.83200 + H 18.05300 -17.76700 10.90200 + H 19.16600 -18.57600 11.53400 + O 19.55100 -5.44900 10.20800 + H 20.50500 -5.51900 10.15600 + H 19.28200 -6.21300 10.71600 + O 18.96900 -10.76400 -0.09800 + H 18.43900 -10.05800 0.27100 + H 19.48000 -10.34500 -0.79100 + O 21.34000 -17.60800 -1.10900 + H 20.94700 -16.78100 -0.82900 + H 22.13900 -17.35100 -1.56900 + O 20.13000 -9.57200 -2.31600 + H 20.67100 -8.80500 -2.50300 + H 20.41700 -10.22900 -2.95100 + O 17.00300 -15.06400 -1.28600 + H 16.78100 -14.13300 -1.26900 + H 17.60700 -15.15300 -2.02400 + O 17.11100 -20.81300 5.80600 + H 17.14400 -21.07900 4.88700 + H 17.95100 -21.09500 6.16700 + O 17.73500 -6.12300 1.60800 + H 16.82700 -5.82400 1.66400 + H 18.06500 -5.73500 0.79700 + O 20.95000 -11.35400 -4.27400 + H 20.98800 -12.30600 -4.18200 + H 21.82700 -11.10700 -4.56800 + O 22.06900 -13.91700 13.07000 + H 21.14600 -13.79400 13.29000 + H 22.06900 -14.10600 12.13100 + O 20.78500 -14.94700 15.93900 + H 20.29400 -14.55600 15.21600 + H 21.69600 -14.70900 15.76600 + O 18.20500 -18.16000 14.76100 + H 18.47900 -18.61400 13.96500 + H 18.62100 -18.64900 15.47100 + O 17.96300 -8.26300 -3.41200 + H 18.69200 -8.68700 -2.95800 + H 17.58100 -8.95600 -3.95100 + O 15.98600 -3.98100 3.86100 + H 15.75400 -4.44000 3.05400 + H 16.20700 -4.67800 4.47800 + O 16.89600 -5.83500 5.62700 + H 16.81400 -6.73700 5.93400 + H 17.35500 -5.38200 6.33500 + O 16.63800 -12.37800 7.45300 + H 16.81600 -12.20900 6.52800 + H 17.42500 -12.82100 7.77100 + O 15.76900 -17.41000 13.75400 + H 16.58300 -17.60500 14.22000 + H 15.32500 -18.25500 13.68300 + O 17.05200 -18.22400 6.72900 + H 17.16900 -19.09200 6.34300 + H 17.89400 -17.78700 6.59900 + O 15.66700 -14.83500 6.27200 + H 16.45300 -15.06600 5.77700 + H 15.89300 -14.01600 6.71300 + O 15.89400 -18.88500 0.95600 + H 15.33500 -18.66300 1.70100 + H 15.64300 -18.26100 0.27400 + O 17.18900 -21.11800 11.07000 + H 16.30700 -21.43400 11.26900 + H 17.06500 -20.49900 10.35000 + O -21.60000 -5.56300 10.08800 + H -21.56700 -4.61800 9.94200 + H -20.68500 -5.81900 10.20800 + O 19.56300 -2.84600 11.13800 + H 18.76600 -2.45700 10.77800 + H 19.49400 -3.77700 10.92600 + O 22.02300 -2.67200 9.83700 + H 21.24700 -2.66100 10.39700 + H 21.71000 -2.36100 8.98800 + O 18.41800 -4.70600 7.77600 + H 18.74400 -4.85200 8.66400 + H 19.07500 -5.11100 7.20900 + O 20.23300 -6.15600 6.28900 + H 20.19900 -7.09400 6.47700 + H 21.15600 -5.92400 6.38900 + O 17.24300 -21.45200 3.11000 + H 16.59600 -21.30900 2.42000 + H 17.48900 -22.37300 3.02100 + O 18.02400 -20.11900 -0.21300 + H 18.32500 -19.52300 -0.89900 + H 17.26400 -19.68000 0.16900 + O 18.18100 -4.83400 -0.84200 + H 17.30600 -4.85000 -1.22900 + H 18.40900 -3.90500 -0.80300 + O 19.23300 -18.84000 -2.35100 + H 19.67500 -19.49500 -2.89200 + H 19.94100 -18.39800 -1.88200 + O 19.08200 -19.88900 12.79000 + H 18.52000 -20.49600 12.30900 + H 19.69700 -20.45200 13.26100 + O 20.90800 -21.36900 14.46700 + H 21.86400 -21.39600 14.47600 + H 20.64100 -22.28800 14.48500 + O 20.02300 -4.93600 -2.96200 + H 19.51700 -4.99000 -2.15200 + H 20.50100 -5.76400 -3.00100 + O -9.86000 7.36200 2.57200 + H -10.23300 8.23300 2.71200 + H -10.51400 6.90500 2.04300 + O -10.44200 6.82800 7.09600 + H -10.27800 6.33900 7.90300 + H -10.99700 6.24900 6.57400 + O -18.34600 6.30600 8.18400 + H -18.23900 5.54300 8.75300 + H -17.46100 6.50100 7.87600 + O -5.94700 9.40800 6.06100 + H -6.60600 8.71300 6.04800 + H -6.34200 10.10500 6.58400 + O -15.97400 10.85900 6.02800 + H -16.40300 10.01100 5.91400 + H -15.04300 10.64900 6.10500 + O -19.86300 5.22400 2.75600 + H -19.84100 5.77900 3.53600 + H -20.68600 4.74100 2.82800 + O -19.61800 6.82000 5.03100 + H -19.32200 6.04100 5.50100 + H -19.00100 7.50500 5.28900 + O -9.68000 13.40600 6.61200 + H -9.32900 12.83200 5.93100 + H -9.62400 12.88800 7.41500 + O -11.63100 9.32000 7.51600 + H -11.24500 8.51800 7.16300 + H -11.10000 9.52400 8.28500 + O -3.30300 0.20500 8.04100 + H -3.44000 1.13800 7.88100 + H -2.60700 -0.04400 7.43200 + O -9.50400 3.32000 7.13500 + H -9.50800 2.39600 7.38700 + H -9.60700 3.31000 6.18400 + O -15.56600 1.15600 4.72900 + H -15.54400 1.96500 4.21700 + H -16.17300 0.58800 4.25500 + O -8.26800 1.65000 3.17900 + H -7.98000 1.26500 4.00700 + H -7.45900 1.93200 2.75200 + O -18.91300 4.18400 5.66500 + H -18.24300 3.77000 6.20900 + H -18.83300 3.75100 4.81500 + O -14.53700 5.47800 5.10000 + H -13.62000 5.27500 5.28500 + H -14.85000 4.73100 4.58900 + O -3.53800 2.57600 10.09100 + H -4.45900 2.66700 9.84800 + H -3.47500 2.98500 10.95300 + O -16.73200 0.95800 9.16800 + H -16.86800 1.49700 8.38800 + H -17.45700 1.18900 9.74800 + O -11.22800 9.77400 2.54500 + H -11.48800 10.19400 1.72500 + H -11.37900 10.44000 3.21500 + O -14.15900 8.60600 8.41800 + H -14.62000 9.36200 8.78100 + H -13.34400 8.96700 8.07000 + O -7.71700 0.90100 5.72500 + H -8.32000 0.64400 6.42200 + H -7.10900 1.50900 6.14500 + O -16.94300 2.40400 6.78300 + H -16.33600 3.02000 7.19500 + H -16.41400 1.94900 6.12700 + O -15.81100 10.53600 9.57900 + H -15.75900 11.48300 9.70800 + H -16.66800 10.39600 9.17600 + O -19.74000 11.15200 1.84700 + H -19.93800 11.78400 1.15500 + H -19.83400 11.64900 2.65900 + O -6.83300 11.37300 7.81500 + H -7.72600 11.56500 8.10200 + H -6.50600 12.20600 7.47700 + O -5.73600 -0.76500 8.93300 + H -6.09900 -1.09200 8.11000 + H -4.88600 -0.39700 8.69100 + O -13.45700 -2.54600 7.33000 + H -13.68000 -3.47600 7.36600 + H -14.28200 -2.09400 7.50900 + O -8.70400 12.18900 4.33700 + H -8.29900 11.36100 4.08000 + H -8.05600 12.85500 4.10700 + O -7.02900 10.20800 3.53000 + H -6.63700 9.95300 2.69500 + H -6.42000 9.87900 4.19100 + O -5.26400 8.30600 11.14500 + H -5.57600 7.81000 10.38800 + H -5.49600 9.21400 10.95000 + O -11.49700 6.29400 0.56100 + H -12.40300 6.56900 0.41800 + H -10.98400 6.84000 -0.03400 + O -10.25800 5.26700 -2.64900 + H -9.72000 5.45800 -1.88100 + H -9.74500 4.63200 -3.15000 + O -4.65300 5.55700 9.18300 + H -5.24700 6.27300 8.95800 + H -4.05900 5.49400 8.43400 + O -7.38600 6.99300 1.48300 + H -8.20000 7.12000 1.97000 + H -6.87600 6.38800 2.02200 + O -8.27300 4.30000 11.30500 + H -8.24300 5.05100 11.89800 + H -8.93400 4.53500 10.65300 + O -16.00900 7.21300 3.48900 + H -16.51200 6.72200 2.84000 + H -15.45800 6.55500 3.91400 + O -13.13500 11.14300 6.04500 + H -12.64400 11.43900 5.27800 + H -12.54500 10.53200 6.48700 + O -15.73300 6.70100 7.23800 + H -15.25000 6.36000 6.48500 + H -15.13400 7.32600 7.64500 + O -9.28700 11.96800 8.94900 + H -9.43200 12.62600 9.62900 + H -9.67600 11.16800 9.30300 + O -14.75100 10.42000 12.19200 + H -15.27300 11.11300 12.59800 + H -15.15200 10.29000 11.33300 + O -8.27600 5.66600 -0.77800 + H -7.93200 6.21600 -0.07500 + H -7.78700 5.93500 -1.55500 + O -17.66000 6.28600 11.90100 + H -17.50800 7.03300 12.47900 + H -18.61300 6.22100 11.83800 + O -11.44700 12.15900 4.01700 + H -10.50400 12.13000 4.18000 + H -11.56800 12.91100 3.43700 + O -16.03100 12.87400 12.94600 + H -15.92900 13.01200 12.00400 + H -15.69000 13.67500 13.34400 + O -5.27500 -2.73500 0.44900 + H -5.93400 -2.37200 1.04200 + H -5.11500 -2.03800 -0.18700 + O -17.70000 6.32600 1.49100 + H -18.09200 7.18800 1.34500 + H -18.41800 5.79100 1.82900 + O -3.17100 11.79300 -1.47500 + H -3.25600 12.73700 -1.34000 + H -2.38300 11.69700 -2.01100 + O -20.78700 8.61800 11.81400 + H -21.48700 8.94800 11.25100 + H -20.69700 7.69700 11.57100 + O -20.27000 10.58800 8.94000 + H -21.09400 10.40100 9.38800 + H -19.81500 11.19900 9.51900 + O -3.39200 12.33300 13.32800 + H -3.41400 11.41300 13.06300 + H -4.22900 12.47500 13.76900 + O -12.22800 11.53000 12.10600 + H -12.33700 12.36500 12.56100 + H -13.09800 11.13100 12.12200 + O -11.20300 14.50400 2.29700 + H -11.35700 14.30700 1.37300 + H -11.62300 15.35300 2.43500 + O -17.70300 14.13500 4.44900 + H -17.37200 13.87900 5.31100 + H -18.39700 13.50300 4.26200 + O -8.57900 14.54200 -1.44400 + H -8.05100 13.85600 -1.85300 + H -8.35500 14.49500 -0.51500 + O -8.51000 14.17100 1.35000 + H -9.36100 14.08900 1.77900 + H -7.87500 13.94300 2.02900 + O -5.81100 3.00200 12.59300 + H -6.50200 3.45400 12.10800 + H -5.79000 2.12200 12.21700 + O -18.74100 10.45500 12.20500 + H -18.11800 9.86100 12.62200 + H -19.50100 9.90700 12.01000 + O -15.96700 -1.37500 7.85400 + H -16.68700 -1.43700 7.22700 + H -16.11000 -0.54200 8.30200 + O -11.91300 -1.63000 11.76900 + H -11.51800 -2.18300 12.44300 + H -12.84800 -1.63300 11.97400 + O -8.53500 1.49900 11.87400 + H -9.02200 1.37200 12.68800 + H -8.55400 2.44400 11.72700 + O -7.13800 6.28900 -3.18100 + H -7.32000 6.11900 -4.10500 + H -6.32000 6.78600 -3.18600 + O -12.28100 0.61800 -2.01100 + H -13.18500 0.81800 -1.77000 + H -12.24800 -0.33800 -2.04300 + O -15.50000 4.17200 8.32400 + H -16.15700 4.12700 9.01900 + H -15.55900 5.07000 7.99800 + O -15.79100 3.63400 3.48600 + H -16.74000 3.51600 3.53000 + H -15.56800 3.44600 2.57400 + O -10.91100 3.89800 1.80200 + H -11.17500 4.72400 1.39800 + H -10.16100 3.60600 1.28400 + O -11.48100 3.74400 12.97600 + H -11.90700 3.14500 13.58900 + H -10.59400 3.84900 13.32000 + O -9.90200 3.50800 4.39200 + H -10.35600 3.88300 3.63800 + H -9.41900 2.76200 4.03600 + O -17.73600 4.22500 9.93400 + H -17.72400 4.77700 10.71600 + H -18.19400 3.43000 10.20800 + O -15.02800 0.39100 -1.82300 + H -15.58300 0.40100 -1.04300 + H -15.48900 0.94400 -2.45300 + O -21.01400 1.05600 4.05800 + H -20.34600 1.06800 4.74300 + H -20.64900 0.49100 3.37700 + O -19.90000 2.50800 1.17600 + H -19.67200 3.01800 0.39900 + H -20.69800 2.92000 1.50800 + O -10.13700 4.87300 9.25600 + H -9.79900 4.33900 8.53700 + H -11.08200 4.72000 9.24500 + O -12.81000 0.72200 4.42900 + H -13.72700 0.80100 4.69200 + H -12.36900 0.36200 5.19900 + O -18.32200 2.60300 3.46500 + H -18.84200 2.76900 2.67900 + H -18.14000 1.66300 3.43800 + O -15.31400 2.65000 0.98300 + H -14.39900 2.43100 1.15800 + H -15.29900 3.07500 0.12500 + O -9.69700 0.67000 7.73400 + H -9.92900 0.52100 8.65000 + H -10.35900 0.18800 7.23700 + O -15.30100 3.70000 -1.53500 + H -15.68100 3.21000 -2.26500 + H -14.45100 3.99600 -1.86100 + O -5.95600 15.93000 14.10700 + H -5.81400 16.13300 13.18200 + H -5.17800 16.27000 14.54900 + O -17.64500 8.41900 13.78200 + H -17.00600 8.47300 14.49200 + H -18.49100 8.34500 14.22400 + O -12.00500 4.76800 5.95900 + H -11.35000 4.22400 5.52100 + H -12.15400 4.33400 6.79800 + O -17.63900 -0.06100 3.19700 + H -18.17100 -0.82800 3.40600 + H -17.44900 -0.14800 2.26200 + O -12.73600 3.89500 8.50100 + H -13.68100 4.02300 8.41500 + H -12.64300 3.27500 9.22400 + O -13.86700 7.58800 -0.11500 + H -14.63000 7.11300 -0.44500 + H -14.21300 8.13400 0.59200 + O -6.25100 7.65300 8.65400 + H -5.62900 8.32300 8.37000 + H -6.85400 7.55600 7.91700 + O -9.82400 0.82000 -0.72200 + H -10.68900 0.95600 -1.11000 + H -9.52400 1.69900 -0.49100 + O -11.70500 8.62900 12.04000 + H -11.31800 8.17800 12.79000 + H -11.84200 9.52600 12.34400 + O -8.71800 3.17000 0.26200 + H -7.89900 2.94300 0.70300 + H -8.54000 4.01100 -0.15800 + O -17.22600 8.39800 5.62400 + H -16.88500 8.11300 4.77600 + H -16.83300 7.79500 6.25500 + O -11.56300 8.92400 -0.90600 + H -11.67800 9.77200 -0.47800 + H -12.39100 8.46700 -0.76100 + O -20.38800 5.85300 12.01200 + H -20.20200 5.50500 12.88400 + H -21.08900 5.29400 11.67500 + O -13.25500 5.85000 13.60900 + H -13.97400 5.65900 13.00600 + H -12.53000 5.30800 13.29800 + O -9.45800 11.66800 12.29800 + H -10.39200 11.61700 12.09600 + H -9.42500 11.83600 13.24000 + O -5.39500 3.57400 -0.92000 + H -5.76700 3.43200 -1.79100 + H -5.11200 4.48800 -0.92700 + O -3.20500 9.75700 12.34600 + H -4.03200 9.31000 12.17000 + H -2.64900 9.53500 11.59900 + O -4.85800 11.85000 0.88000 + H -5.42700 11.08100 0.87600 + H -4.41000 11.82100 0.03400 + O -7.76000 7.30500 6.36700 + H -8.70800 7.29500 6.49800 + H -7.57700 6.50600 5.87300 + O -15.12600 11.36900 3.39900 + H -15.09900 12.31400 3.24600 + H -15.48700 11.28000 4.28100 + O -20.48600 6.73800 -0.22800 + H -19.98500 7.42700 0.20900 + H -21.18200 6.51800 0.39200 + O -14.95600 9.15600 1.86300 + H -14.91200 9.94000 2.41000 + H -15.32600 8.48400 2.43700 + O -12.79400 13.17400 9.78900 + H -12.27100 13.97400 9.73900 + H -12.34600 12.63400 10.44100 + O -15.34100 14.08900 3.03700 + H -16.23400 14.12600 3.38000 + H -14.91100 14.86100 3.40500 + O -10.21700 9.51400 9.88600 + H -10.86700 9.18300 10.50500 + H -9.37700 9.21700 10.23600 + O -6.78500 9.92700 -1.80800 + H -6.63700 9.74600 -0.88000 + H -7.59800 9.46600 -2.01600 + O -7.07000 13.09300 11.60600 + H -7.98700 12.82900 11.68800 + H -7.07600 13.77600 10.93600 + O -3.58100 14.45500 -0.89800 + H -3.64000 15.06700 -1.63200 + H -4.07700 14.87500 -0.19500 + O -18.86600 1.78700 10.71400 + H -18.86900 1.79900 11.67200 + H -19.62600 1.25500 10.47800 + O -6.15100 9.33700 0.85600 + H -6.73600 8.60600 1.05500 + H -5.27900 8.94400 0.82400 + O -19.09700 12.74100 10.62700 + H -18.75700 12.07900 11.22900 + H -18.84100 13.57600 11.01900 + O -13.95000 10.65400 -2.79700 + H -14.00000 9.78100 -3.18500 + H -14.27500 10.53800 -1.90400 + O -15.13300 10.78000 -0.33400 + H -15.07100 10.21600 0.43700 + H -14.92300 11.65500 -0.00900 + O -18.91700 8.73600 0.82400 + H -19.24700 9.47200 1.33900 + H -18.38900 9.14200 0.13700 + O -10.52400 0.31400 10.36600 + H -9.75400 0.55900 10.87900 + H -10.94500 -0.37500 10.87900 + O -9.57700 -0.30800 1.77500 + H -9.68500 0.00700 0.87800 + H -9.24400 0.45000 2.25700 + O -11.99000 11.40800 0.37900 + H -12.82800 11.83600 0.55700 + H -11.41000 12.12000 0.10900 + O -12.82100 1.90300 1.92900 + H -12.15500 2.58500 1.84800 + H -12.74600 1.60100 2.83500 + O -12.42500 2.30200 10.70500 + H -12.09200 2.83800 11.42500 + H -11.74200 1.64900 10.55600 + O -11.83600 -1.40400 2.94000 + H -12.10500 -0.61700 3.41300 + H -11.00900 -1.16200 2.52200 + O -14.19600 13.17900 0.68900 + H -14.02200 14.01600 0.25700 + H -14.62700 13.41800 1.50900 + O -16.61200 0.24900 0.50600 + H -16.43800 1.16300 0.73000 + H -15.82700 -0.22000 0.78800 + O -5.99500 0.40400 11.53500 + H -6.92900 0.60400 11.59500 + H -5.87900 0.06600 10.64700 + O -7.39900 4.75100 5.18500 + H -8.27600 4.42100 4.99000 + H -7.03300 4.11100 5.79600 + O -17.50300 9.79700 -1.30000 + H -16.74900 10.29700 -0.98800 + H -17.13100 9.14800 -1.89700 + O -19.62000 4.17200 -0.97100 + H -19.90200 5.05500 -0.73500 + H -20.00900 4.01700 -1.83200 + O -5.91800 13.85500 7.05000 + H -6.27200 14.51200 6.45200 + H -4.96800 13.93100 6.95500 + O -18.43000 -1.50400 6.46300 + H -18.60500 -1.66700 5.53600 + H -18.71400 -2.30400 6.90500 + O -20.07300 12.75600 4.16300 + H -20.27300 12.35900 5.01100 + H -20.63900 13.52600 4.12000 + O -19.91900 14.11100 15.87500 + H -19.22600 14.68900 16.19300 + H -19.47100 13.49700 15.29300 + O -12.01200 12.61000 -2.73800 + H -12.68800 11.93200 -2.72300 + H -11.20600 12.14000 -2.95500 + O -9.49800 11.90400 15.02300 + H -9.54800 11.00900 15.35900 + H -8.77500 12.30200 15.50800 + O -7.20200 12.52400 -2.65200 + H -6.87400 11.69200 -2.31100 + H -6.60100 12.74500 -3.36300 + O -20.37100 8.03100 14.51300 + H -20.09200 7.11700 14.48100 + H -20.53500 8.26500 13.59900 + O -16.77900 7.74000 -3.02600 + H -16.57900 7.14700 -2.30300 + H -15.93800 7.88500 -3.45800 + O -13.92000 8.55400 14.12900 + H -14.16000 8.97200 13.30200 + H -13.57400 7.69900 13.87400 + O -9.29300 8.89400 -2.44200 + H -10.05200 8.82000 -1.86300 + H -9.49000 8.30500 -3.17100 + O -8.18400 9.22200 11.82100 + H -7.41100 9.56600 11.37500 + H -8.65600 10.00000 12.11800 + O -19.36400 5.46100 14.56600 + H -18.53500 5.00100 14.43600 + H -19.63900 5.21300 15.44900 + O -16.21500 6.21200 -0.83500 + H -16.73400 6.16300 -0.03200 + H -15.98200 5.30300 -1.02600 + O -15.16300 5.11300 11.77900 + H -16.03700 5.50100 11.81200 + H -14.65700 5.70800 11.22700 + O -12.96900 4.65600 -2.51800 + H -13.12500 5.13900 -3.33000 + H -12.04200 4.80000 -2.32800 + O -10.41900 7.32800 14.22400 + H -9.62300 7.00400 13.80200 + H -10.68600 6.61900 14.80900 + O -4.69500 7.52200 -3.46700 + H -4.52500 7.15900 -4.33600 + H -4.49800 8.45500 -3.55400 + O -14.84700 2.77500 13.22200 + H -14.91300 2.03000 12.62400 + H -14.85400 3.54300 12.65000 + O -14.79000 0.89800 11.17200 + H -14.00600 1.35300 10.86600 + H -15.39900 0.94800 10.43600 + O -13.63900 7.07500 10.68000 + H -12.92900 7.55800 11.10300 + H -13.80400 7.55500 9.86800 + O -9.19300 3.57800 14.62200 + H -9.44400 2.65800 14.53500 + H -8.27900 3.55100 14.90500 + O -7.88700 6.70400 12.99700 + H -8.00800 7.54200 12.55000 + H -6.93800 6.58600 13.03500 + O -17.17700 3.71000 14.39600 + H -17.80000 3.05800 14.07500 + H -16.34200 3.45800 14.00100 + O -14.68900 -1.74100 12.04600 + H -14.80000 -0.80400 11.88000 + H -15.29500 -2.16600 11.43800 + O -12.63000 1.97000 14.77700 + H -12.79600 1.61700 15.65100 + H -13.50000 2.16100 14.42600 + O -10.08200 1.05900 14.10000 + H -11.01300 1.17400 14.29000 + H -9.81400 0.32400 14.65100 + O -9.26500 -0.86300 15.86400 + H -9.33400 -1.81700 15.82200 + H -9.62700 -0.63500 16.72100 + O -17.63100 -0.38800 14.56900 + H -17.66700 -1.30800 14.83100 + H -17.20400 0.05400 15.30200 + O -15.09700 -3.19400 14.37700 + H -14.41900 -3.86600 14.30400 + H -15.04500 -2.71100 13.55300 + O -14.56000 -1.35900 16.34100 + H -14.75900 -1.92200 15.59300 + H -15.28400 -0.73400 16.37100 + O -13.38700 15.67500 -0.28300 + H -12.54100 15.26800 -0.47000 + H -13.55500 16.23800 -1.03800 + O -19.72300 10.16600 -2.82800 + H -18.97300 10.25300 -2.24000 + H -19.85700 9.22200 -2.90700 + O 22.25800 15.02300 4.55500 + H 21.48100 14.64100 4.14500 + H 21.95700 15.31100 5.41700 + O -20.72600 12.36700 6.84000 + H -20.53700 11.63500 7.42700 + H -20.26000 13.10900 7.22500 + O -3.14100 13.98400 7.20700 + H -2.82600 14.65800 7.80900 + H -2.34500 13.57100 6.87200 + O -3.85600 13.87100 11.01300 + H -3.89500 13.17900 10.35300 + H -3.66700 13.41200 11.83200 + O -5.08200 10.14100 15.46700 + H -5.94900 10.12600 15.87300 + H -4.61000 10.82900 15.93700 + O -3.73200 8.11200 0.76600 + H -3.05500 8.78800 0.80400 + H -3.46700 7.47100 1.42500 + O -3.28600 5.98800 2.45100 + H -4.17400 5.70700 2.67000 + H -2.81800 5.97700 3.28600 + O -4.06400 9.34900 8.20100 + H -4.36200 9.43600 7.29600 + H -3.94700 10.24900 8.50300 + O -6.24900 10.71700 10.48000 + H -6.37900 11.57200 10.89000 + H -6.28400 10.89300 9.54000 + O -4.45600 6.19200 -1.09500 + H -4.63900 6.68500 -1.89500 + H -4.36600 6.86100 -0.41700 + O -6.16400 2.80100 -3.50700 + H -5.81700 2.94200 -4.38700 + H -6.15100 1.85100 -3.39200 + O -5.89300 5.07300 2.89600 + H -5.97800 4.20100 2.51100 + H -6.35000 5.01000 3.73500 + O -20.71900 7.62500 8.53500 + H -20.50000 8.55600 8.49100 + H -19.88700 7.17600 8.38600 + O -5.09600 6.57600 13.21700 + H -4.18900 6.31500 13.37600 + H -5.04900 7.15100 12.45200 + O -3.17300 3.74400 12.70800 + H -4.12000 3.63700 12.80300 + H -2.80300 2.94800 13.08900 + O -6.11800 0.04000 -3.54700 + H -5.69800 -0.34400 -2.77800 + H -7.04300 -0.18800 -3.45600 + O -6.12500 2.55900 1.55600 + H -5.57800 1.79600 1.74300 + H -5.80800 2.88100 0.71200 + O -6.10900 2.83600 6.82000 + H -5.16400 2.75300 6.69100 + H -6.21400 2.93000 7.76700 + O -6.20600 3.37200 9.55400 + H -7.02200 3.62600 9.98600 + H -5.71900 4.19100 9.46100 + O -18.88400 1.74800 13.45600 + H -18.46200 0.93300 13.72800 + H -19.50300 1.94500 14.15900 + O -3.46100 -2.57700 15.40700 + H -2.72800 -3.15700 15.19800 + H -3.50100 -1.96700 14.67000 + O -4.73500 0.20500 2.25000 + H -4.13300 -0.00200 2.96500 + H -5.42200 -0.45800 2.31600 + O -19.42500 1.09400 6.39900 + H -19.18000 0.18200 6.55100 + H -18.61200 1.58500 6.52100 + O -21.22200 3.31100 7.00200 + H -20.60600 3.83800 6.49100 + H -20.81200 2.44800 7.05500 + O -4.07000 -0.94100 13.18000 + H -4.72500 -0.35700 12.79700 + H -4.02900 -1.68300 12.57700 + O -4.54400 -0.88400 -1.47000 + H -3.97800 -1.41000 -2.03400 + H -4.09600 -0.04100 -1.39900 + O -20.37200 -0.15300 1.54400 + H -20.94300 -0.36600 0.80600 + H -20.11100 0.75500 1.39100 + O -6.82900 14.03700 3.56000 + H -5.91200 13.84900 3.36000 + H -6.79500 14.77000 4.17500 + O -20.96200 0.05100 10.15300 + H -21.29300 -0.84600 10.12000 + H -21.73700 0.59800 10.02500 + O -18.57900 12.58400 14.02600 + H -18.81300 11.80400 13.52300 + H -17.66400 12.75000 13.79900 + O -15.48400 13.16800 10.26400 + H -15.77300 14.07100 10.13400 + H -14.54200 13.19000 10.09600 + O -12.33000 14.03300 13.22300 + H -12.22700 13.80700 14.14700 + H -11.62700 14.66000 13.04900 + O -10.23900 15.72400 12.80600 + H -9.71100 15.26000 12.15700 + H -9.63900 15.88000 13.53600 + O -4.23300 -3.03600 11.36500 + H -5.13200 -3.09700 11.04200 + H -3.87300 -3.91400 11.23800 + O -18.72800 15.45100 11.33700 + H -19.42200 15.92900 10.88300 + H -18.82600 15.70200 12.25600 + O -16.23800 15.76900 10.09000 + H -17.11900 15.78200 10.46600 + H -16.30300 16.31700 9.30800 + O -11.67200 15.65300 9.24600 + H -12.16400 15.94300 8.47800 + H -11.68400 16.40600 9.83700 + O -9.47500 14.42000 10.36800 + H -10.17800 14.86200 9.89300 + H -8.67300 14.70000 9.92700 + O -6.97900 14.91600 9.35400 + H -6.54300 15.76200 9.25300 + H -6.69500 14.40800 8.59400 + O -18.98100 14.16800 8.10200 + H -19.01100 13.78600 8.97900 + H -18.14100 13.88100 7.74400 + O -16.66000 13.32100 6.89900 + H -16.55900 12.41000 6.62500 + H -15.77400 13.60300 7.12700 + O -13.90500 13.51100 7.28800 + H -13.66800 12.76000 6.74400 + H -13.49800 13.33100 8.13500 + O -11.52600 -0.82800 6.32500 + H -12.12100 -1.46300 6.72300 + H -10.77500 -1.34800 6.04000 + O -6.46200 -1.95900 6.51400 + H -5.85700 -1.91600 5.77400 + H -7.32200 -1.78600 6.13100 + O -4.26800 -2.55100 4.54400 + H -3.71300 -3.26900 4.23800 + H -3.73900 -1.76500 4.40800 + O -19.13000 -2.33900 3.91700 + H -19.98400 -2.76100 3.82500 + H -18.50800 -3.06500 3.96700 + O -19.22400 -2.47000 0.41900 + H -18.99900 -1.90000 -0.31600 + H -19.54200 -1.87300 1.09700 + O -13.66100 -2.91000 4.45100 + H -13.01600 -2.35800 4.00900 + H -13.52500 -2.73900 5.38300 + O -9.27300 -2.27000 5.66400 + H -9.13300 -2.55500 4.76000 + H -9.36400 -3.08400 6.16000 + O -7.09500 -1.60700 2.20900 + H -7.85400 -1.08200 1.95600 + H -7.46700 -2.36000 2.66800 + O -20.42000 12.86900 -0.26700 + H -21.01900 12.43000 -0.87100 + H -20.80600 13.73400 -0.13200 + O -18.43200 -0.43800 -1.54900 + H -17.87300 -0.96900 -2.11600 + H -17.85200 -0.14800 -0.84500 + O -14.09100 -0.49300 1.22600 + H -13.52400 -1.18500 1.56800 + H -13.59600 0.31400 1.36800 + O -11.16400 14.05200 -0.57800 + H -10.28400 14.31000 -0.85400 + H -11.53500 13.61000 -1.34200 + O -4.15600 13.72300 2.75100 + H -4.35200 13.02000 2.13200 + H -4.25100 14.52600 2.23900 + O -17.41700 -2.82400 15.87800 + H -16.64400 -3.12500 15.40100 + H -17.27200 -3.11300 16.77800 + O -18.29900 13.74800 -1.97200 + H -18.73400 13.34700 -1.21900 + H -17.59600 13.13800 -2.19600 + O -8.77900 -3.56600 15.45100 + H -8.72100 -3.95800 16.32300 + H -7.87100 -3.38600 15.20800 + O -5.91400 13.18900 14.11800 + H -6.32800 13.03100 13.26900 + H -5.91500 14.14200 14.21200 + O 7.32300 7.78700 4.01100 + H 7.18400 8.59400 3.51600 + H 7.41600 7.10800 3.34300 + O -0.31100 5.29300 8.12600 + H -0.11000 4.76000 8.89500 + H 0.52500 5.37700 7.66700 + O 12.23300 9.28300 5.54500 + H 11.95900 8.48600 6.00000 + H 12.15100 9.97400 6.20300 + O 1.87500 9.50600 6.09700 + H 1.38000 8.78800 5.70200 + H 2.02800 9.22400 6.99900 + O -1.61500 4.51900 0.80500 + H -2.24700 5.00800 1.33100 + H -1.86200 4.70600 -0.10100 + O -1.67700 5.78500 4.69200 + H -1.39800 4.87200 4.77200 + H -0.87900 6.29300 4.83600 + O 9.20500 13.10700 6.70000 + H 9.43500 12.65100 5.89100 + H 9.15900 12.41500 7.36000 + O 15.53000 -0.28200 8.84900 + H 15.36900 0.46000 9.43200 + H 15.97000 0.10000 8.09000 + O 8.85600 3.27300 7.17500 + H 8.68500 2.34900 7.36000 + H 9.12800 3.28700 6.25700 + O 2.21900 0.37000 3.77800 + H 2.13000 1.09600 3.16100 + H 1.56300 -0.26700 3.49500 + O 10.70400 0.73500 4.00800 + H 10.89300 0.47400 4.90900 + H 11.48400 0.48100 3.51600 + O -1.11400 3.04400 4.85300 + H -0.42400 2.56800 5.31400 + H -1.03400 2.75800 3.94200 + O 3.25900 3.29000 5.24500 + H 4.05000 2.76500 5.12400 + H 2.81700 3.25800 4.39600 + O 15.24200 1.68200 10.73500 + H 14.45800 2.09300 10.37200 + H 15.26400 1.96900 11.64800 + O 1.18000 0.39600 8.78700 + H 1.52100 1.20700 8.41100 + H 0.32600 0.63700 9.14500 + O 6.92800 9.94100 2.40400 + H 6.84500 10.19300 1.48500 + H 7.02800 10.76900 2.87300 + O 5.01300 8.32000 8.76000 + H 4.25700 8.86700 8.96900 + H 5.56600 8.36500 9.54100 + O 11.08600 0.21000 6.66500 + H 10.21300 0.27700 7.05000 + H 11.56600 0.95500 7.02700 + O 0.99200 1.56500 5.91600 + H 1.68500 2.14900 6.22400 + H 1.40900 1.03600 5.23600 + O 2.78000 9.74000 9.57600 + H 2.82700 10.69500 9.60700 + H 1.90300 9.53200 9.90000 + O -1.92300 10.19300 0.45600 + H -2.43200 10.66500 -0.20300 + H -1.43500 10.87600 0.91700 + O 11.82400 11.37700 7.40000 + H 11.01700 11.46800 7.90700 + H 12.02600 12.26700 7.11000 + O 13.43800 -2.01300 9.28100 + H 13.16200 -2.10800 8.36900 + H 14.16700 -1.39400 9.24700 + O 4.55500 -3.07200 7.14100 + H 4.36400 -4.01000 7.13000 + H 3.72200 -2.66000 7.37100 + O 9.94800 12.14000 4.20000 + H 10.30400 11.33500 3.82300 + H 10.53200 12.83000 3.88700 + O 11.29600 10.02000 3.08600 + H 11.87700 9.71200 2.39100 + H 11.71400 9.72600 3.89600 + O 13.06000 8.94600 10.61200 + H 13.23400 8.45400 9.80900 + H 13.17500 9.86300 10.36100 + O 7.52600 6.34500 1.54800 + H 6.63700 6.44000 1.20600 + H 8.05700 6.92100 0.99700 + O 8.72500 4.72100 -2.81200 + H 9.04400 4.88800 -1.92500 + H 9.42500 4.21800 -3.22700 + O 14.46300 5.29600 9.67600 + H 14.07800 6.12100 9.38000 + H 15.13300 5.09600 9.02300 + O 11.40800 6.51700 1.78500 + H 10.80200 6.79900 2.47000 + H 12.04000 5.95900 2.23800 + O 10.90300 3.78700 11.99600 + H 10.94900 4.63400 12.43900 + H 10.32700 3.93800 11.24700 + O 2.10800 6.07400 2.84100 + H 1.64100 6.12300 2.00700 + H 2.27100 5.14000 2.97000 + O 5.86600 10.59200 5.44000 + H 5.16700 10.33400 4.84000 + H 6.41900 11.18000 4.92600 + O 2.16000 5.26800 6.83000 + H 2.71100 4.72400 6.26700 + H 2.75300 5.58600 7.51100 + O 9.71100 11.70900 9.18400 + H 9.54300 12.59400 9.50500 + H 9.58200 11.14700 9.94800 + O 4.00300 9.83300 12.14200 + H 3.37200 10.45800 12.50000 + H 3.62400 9.56100 11.30600 + O 10.33900 5.14000 -0.44900 + H 10.63200 5.70200 0.26900 + H 11.05100 5.17500 -1.08700 + O 0.07200 5.87800 11.89200 + H 0.20500 6.64900 12.44400 + H -0.81000 5.98600 11.53700 + O 7.25100 12.31700 3.73100 + H 8.18800 12.32100 3.92600 + H 7.11900 13.08200 3.17200 + O 2.15900 11.78600 12.66000 + H 2.32000 12.14600 11.78800 + H 1.20600 11.77900 12.74500 + O 14.14800 -3.01900 -0.23000 + H 13.28800 -2.77100 0.10800 + H 14.44600 -2.25100 -0.71700 + O 0.74900 5.92800 0.48200 + H 0.33800 6.72100 0.13600 + H 0.02200 5.31900 0.61400 + O 15.22600 11.23600 -1.93500 + H 15.21900 12.19200 -1.89900 + H 16.15500 11.00300 -1.92700 + O -2.11600 8.49400 10.11200 + H -2.71900 8.67400 9.39100 + H -2.26100 7.57100 10.32200 + O -3.43300 11.92600 9.04500 + H -2.50400 11.83300 9.25600 + H -3.45400 12.55900 8.32700 + O 15.54000 11.29100 12.72400 + H 15.45100 10.34200 12.63500 + H 14.76700 11.56000 13.22000 + O 6.41400 11.07600 11.73900 + H 6.42700 11.91300 12.20400 + H 5.57100 10.68600 11.97100 + O 7.26800 14.46800 1.83700 + H 6.57700 14.46600 1.17500 + H 7.19400 15.32500 2.25800 + O 0.91600 14.03200 4.33200 + H 1.20900 13.44200 5.02600 + H 0.02700 13.74000 4.12800 + O 10.01600 14.19000 -1.85900 + H 10.56700 13.43900 -2.07600 + H 10.08300 14.27000 -0.90700 + O 9.90900 14.30100 0.94500 + H 8.99000 14.23500 1.20200 + H 10.39600 14.01900 1.71900 + O 12.41900 2.27600 13.94300 + H 11.93400 2.88600 13.38800 + H 12.36700 1.43800 13.48200 + O 0.33300 9.52200 10.81100 + H 0.43400 9.19200 11.70400 + H -0.47300 9.11500 10.49400 + O 2.10400 -1.98300 7.79200 + H 1.38000 -2.26900 7.23600 + H 1.84200 -1.11500 8.09900 + O 6.92500 -1.99300 11.32700 + H 7.40000 -2.47300 12.00600 + H 6.02200 -1.96000 11.64400 + O 9.63900 1.28500 12.25700 + H 9.30100 1.27400 13.15200 + H 9.99200 2.16800 12.14400 + O 12.23100 4.90100 -2.40400 + H 11.75800 4.53500 -3.15200 + H 12.86000 5.50900 -2.79300 + O 7.80000 -0.81200 -1.62800 + H 7.19200 -0.53800 -0.94100 + H 7.26700 -1.33800 -2.22400 + O 2.23300 2.91600 8.39500 + H 1.67700 3.13800 9.14200 + H 2.13600 3.65500 7.79500 + O 1.98000 3.16000 2.77900 + H 1.06600 2.95900 2.57800 + H 2.44600 2.99700 1.95900 + O 7.93200 3.72200 2.38200 + H 7.79500 4.62700 2.10400 + H 8.55500 3.36800 1.74800 + O 6.82400 3.72900 13.09300 + H 6.31600 3.03700 13.51800 + H 7.57700 3.86300 13.66900 + O 9.59400 3.17900 4.53800 + H 8.94800 3.41400 3.87100 + H 9.92900 2.32800 4.25800 + O 0.36500 3.60300 10.23800 + H 0.18400 4.18100 10.97900 + H -0.19300 2.84000 10.38700 + O 4.58900 0.46300 -3.22000 + H 4.27300 0.35800 -2.32300 + H 3.90300 0.96700 -3.65800 + O -2.54600 -0.45300 3.95500 + H -2.22500 -0.21300 4.82400 + H -1.75300 -0.58400 3.43400 + O -3.43800 1.65200 -1.36600 + H -2.93700 2.11200 -2.03900 + H -4.13800 2.26100 -1.13000 + O 9.22900 3.98400 9.85500 + H 9.29100 3.83800 8.91100 + H 8.32600 3.75500 10.07400 + O 4.94200 0.72600 4.37800 + H 4.02900 0.44600 4.31600 + H 5.27000 0.30300 5.17200 + O -0.61500 2.37300 2.17700 + H -0.97000 3.10600 1.67400 + H -1.02700 1.59800 1.79400 + O 3.45700 2.51600 0.58000 + H 4.33800 2.48200 0.95300 + H 3.58200 2.87200 -0.29900 + O 8.41700 0.58700 7.61800 + H 8.25100 0.47600 8.55400 + H 7.69600 0.12700 7.18900 + O 3.67700 3.36000 -2.03900 + H 3.21900 2.94700 -2.77100 + H 4.50400 3.66100 -2.41400 + O 13.63300 15.19200 14.12900 + H 13.85600 15.42700 13.22800 + H 14.36400 15.52300 14.65100 + O 0.44300 8.24300 13.36300 + H 1.07600 8.11400 14.06900 + H -0.35000 8.55000 13.80400 + O 6.79700 5.11600 7.14600 + H 7.55600 4.53300 7.12700 + H 6.10300 4.59600 7.55100 + O 0.06100 -1.16000 2.91000 + H -0.03200 -2.04400 3.26300 + H 0.02600 -1.27600 1.96000 + O 4.97500 3.64200 8.54600 + H 4.20400 3.07500 8.56300 + H 5.55100 3.28300 9.22100 + O 5.15700 7.04000 0.29500 + H 4.52300 6.59600 -0.26800 + H 4.62300 7.58000 0.87800 + O 13.66200 7.57500 8.36600 + H 14.43500 7.82900 7.86100 + H 12.99700 7.38100 7.70500 + O 9.71400 -0.06100 0.27600 + H 9.10500 -0.24900 -0.43800 + H 9.71100 0.89300 0.34900 + O 6.75400 8.40100 10.91200 + H 6.98400 7.87500 11.67700 + H 6.69400 9.29600 11.24500 + O 9.93900 2.70200 0.77600 + H 10.77400 2.62100 1.23800 + H 10.04000 3.48300 0.23200 + O 0.87600 7.03200 5.11700 + H 1.29300 6.81600 4.28300 + H 1.27000 6.42800 5.74600 + O 7.31300 8.45300 -0.69300 + H 7.07600 9.36600 -0.52600 + H 6.53000 7.95500 -0.45700 + O -2.58100 5.84000 10.96400 + H -2.71300 5.11800 11.57800 + H -3.28000 5.73600 10.31800 + O 4.73800 5.57800 13.57200 + H 4.04100 5.39200 12.94300 + H 5.48000 5.05500 13.27000 + O 9.15400 10.63400 11.78300 + H 8.21200 10.76900 11.67900 + H 9.30100 10.69100 12.72800 + O 15.50600 3.43000 -0.36300 + H 14.59800 3.15400 -0.48700 + H 15.53400 4.32200 -0.71000 + O 15.34200 8.44300 12.01300 + H 14.51600 8.62900 11.56600 + H 15.94200 8.18900 11.31300 + O 13.78300 11.42400 0.46900 + H 13.35000 10.59800 0.68400 + H 14.21100 11.26000 -0.37100 + O 11.52800 6.85300 6.76700 + H 10.73000 6.69700 7.27300 + H 11.69200 6.02500 6.31600 + O 3.53600 10.32900 3.93000 + H 3.40500 11.24300 3.67900 + H 2.88500 10.17300 4.61500 + O -1.81300 5.81100 -1.72700 + H -1.42400 6.60900 -1.36900 + H -2.75600 5.93500 -1.61900 + O 3.42300 8.44600 1.89200 + H 3.55800 9.10100 2.57700 + H 2.98800 7.71900 2.33800 + O 5.73700 12.48400 9.41300 + H 6.25900 13.28200 9.33500 + H 6.17400 11.97700 10.09700 + O 3.32300 13.05500 3.20800 + H 2.47800 13.45800 3.40800 + H 3.96800 13.72000 3.44500 + O 8.70500 7.98800 8.97800 + H 7.99800 8.07400 9.61800 + H 9.49900 7.91400 9.50700 + O 11.75100 9.55600 -1.58600 + H 11.95200 9.29400 -0.68700 + H 10.93000 9.10800 -1.78800 + O 11.77300 13.21700 11.35000 + H 10.81900 13.15600 11.40100 + H 11.93600 13.98800 10.80700 + O 15.24500 13.97600 -1.59500 + H 15.65800 14.56200 -2.22900 + H 14.52900 14.49100 -1.22400 + O -1.13800 1.27400 10.11500 + H -1.95200 1.74700 9.93900 + H -1.42100 0.39900 10.38200 + O 12.77400 8.77300 0.92000 + H 12.25900 8.00100 1.15500 + H 13.67900 8.46200 0.90400 + O -0.70800 11.89400 9.57500 + H -0.29700 11.17700 10.05700 + H -0.33200 12.68700 9.95800 + O 4.47600 10.19100 -3.02600 + H 4.49300 9.35600 -3.49300 + H 4.18000 9.96700 -2.14400 + O 3.32100 9.98000 -0.47800 + H 3.31300 9.40800 0.29000 + H 3.53400 10.84600 -0.13000 + O -0.62000 8.04100 -0.64200 + H -1.09500 8.72300 -0.16600 + H -0.01800 8.52100 -1.21100 + O 7.91900 0.36000 10.32900 + H 8.58700 0.59400 10.97400 + H 7.56500 -0.47300 10.64100 + O 8.86800 -0.89000 2.81200 + H 9.12900 -0.78400 1.89800 + H 9.43200 -0.28600 3.29400 + O 6.54700 11.07700 -0.10700 + H 5.69000 11.42000 0.14500 + H 6.99000 11.81700 -0.52200 + O 5.77600 2.03100 2.04500 + H 6.49600 2.63700 2.21900 + H 5.53900 1.68700 2.90700 + O 6.73200 2.87000 10.47000 + H 6.72100 3.13300 11.39000 + H 7.04900 1.96700 10.48200 + O 6.19800 -1.40900 2.91700 + H 5.80200 -0.67500 3.38700 + H 7.14100 -1.26700 3.00100 + O 4.20500 12.37700 0.64400 + H 4.43500 13.24600 0.31600 + H 3.85900 12.53600 1.52200 + O 3.49200 0.04800 -0.72300 + H 3.25800 0.83100 -0.22500 + H 4.31100 -0.25200 -0.32800 + O 12.09200 0.00100 12.37000 + H 11.16800 0.23200 12.27200 + H 12.34200 -0.35600 11.51800 + O 11.83200 4.32600 5.65900 + H 11.03600 3.95500 5.27800 + H 12.13100 3.66300 6.28100 + O 1.13400 9.51600 -2.13400 + H 1.84000 9.82700 -1.56800 + H 1.53300 8.82000 -2.65600 + O -2.34500 3.43600 -3.11600 + H -2.02400 4.24900 -2.72600 + H -2.52900 3.66400 -4.02800 + O 12.73700 14.02000 7.07400 + H 12.59900 14.66300 6.37800 + H 13.68200 14.02900 7.22600 + O -0.16000 -2.73300 6.42700 + H -0.19800 -3.02500 5.51700 + H -0.47800 -3.48100 6.93300 + O -0.83700 12.56300 1.38700 + H -1.08100 12.71000 2.30100 + H -0.75800 13.44200 1.01500 + O -1.92700 14.49800 14.36600 + H -1.39500 14.07600 15.04100 + H -2.48600 13.79900 14.02800 + O 2.43900 12.51900 15.31900 + H 3.14600 11.99900 15.70000 + H 2.41000 12.24800 14.40100 + O 6.53800 11.90900 -3.64400 + H 5.87000 11.26300 -3.41400 + H 7.34600 11.40200 -3.72000 + O 9.11500 10.77900 14.57100 + H 9.10800 9.99900 15.12700 + H 9.80800 11.32800 14.93700 + O 11.45500 12.04000 -2.74800 + H 11.69900 11.25300 -2.26100 + H 12.19500 12.19900 -3.33400 + O -1.99300 8.73200 14.59700 + H -2.23500 7.81200 14.49300 + H -2.47800 9.18700 13.90900 + O 2.30900 7.31600 -3.28100 + H 2.65200 6.81200 -2.54400 + H 3.06500 7.44900 -3.85400 + O 4.52100 8.25900 14.36300 + H 4.45000 8.74000 13.53900 + H 4.73800 7.36400 14.10200 + O 9.39500 8.44700 -2.44600 + H 8.67500 8.36100 -1.82100 + H 9.15400 7.87000 -3.17000 + O 10.50400 8.22200 11.27100 + H 11.39700 8.50500 11.07500 + H 10.03700 9.03000 11.48400 + O -2.56800 5.99700 14.22100 + H -2.59500 5.15500 13.76600 + H -2.26500 5.78400 15.10300 + O 3.05200 5.85700 -1.06200 + H 2.27400 5.79100 -0.50800 + H 3.20800 4.96300 -1.36500 + O 2.75100 5.07000 11.66600 + H 1.84800 5.36400 11.78900 + H 2.98200 5.36900 10.78700 + O 5.92300 4.38700 -3.19600 + H 5.77800 4.84400 -4.02500 + H 6.85900 4.49500 -3.02400 + O 7.61500 7.16500 13.23600 + H 8.43100 6.67400 13.14000 + H 7.32100 6.97100 14.12600 + O 14.01400 6.66100 -3.57300 + H 13.91500 6.48200 -4.50800 + H 13.93800 7.61200 -3.50100 + O 2.73400 2.54700 13.00600 + H 2.56400 1.84100 12.38300 + H 2.71900 3.34400 12.47600 + O 2.29100 0.29000 11.34500 + H 2.17600 0.42600 10.40500 + H 1.55500 -0.26600 11.60000 + O 3.61900 5.93500 9.08400 + H 4.25000 5.22300 8.98100 + H 4.14200 6.73300 9.00400 + O 8.90100 4.00400 14.92200 + H 9.00400 3.05500 14.99700 + H 9.70100 4.36200 15.30800 + O 11.23400 6.28700 13.16300 + H 10.96100 6.95300 12.53200 + H 12.18400 6.38500 13.22100 + O 1.93100 2.33900 15.66000 + H 1.09500 1.87900 15.58400 + H 2.16000 2.57000 14.76000 + O 4.14900 -1.85600 11.71800 + H 3.74900 -0.98600 11.71900 + H 3.56500 -2.38800 11.17700 + O 5.18700 1.87800 14.16300 + H 4.97400 1.33000 14.91800 + H 4.34100 2.07700 13.76300 + O 9.02900 1.27000 14.94400 + H 8.28600 0.94500 15.45200 + H 9.70600 0.60200 15.05500 + O 10.70400 -0.83400 15.41500 + H 10.67900 -1.77100 15.22100 + H 10.59700 -0.78200 16.36400 + O -0.35400 0.65800 15.68500 + H -0.34900 -0.06800 15.06100 + H 0.01800 0.28900 16.48600 + O 3.96000 -3.29400 14.05200 + H 4.54800 -4.03900 13.93400 + H 3.99900 -2.82200 13.22000 + O 5.36600 -1.78300 15.85200 + H 4.91500 -2.36000 15.23500 + H 4.79300 -1.02000 15.93000 + O 5.37700 14.76400 -0.28300 + H 6.06700 14.33000 -0.78500 + H 5.04500 15.44400 -0.87000 + O -1.05600 10.92600 -3.06700 + H -0.26200 10.56800 -2.67000 + H -1.35500 10.24100 -3.66500 + O -1.68900 13.13400 3.97000 + H -2.56200 13.38600 3.66800 + H -1.83600 12.74900 4.83400 + O 15.39200 14.03300 7.90000 + H 15.47400 13.81600 8.82900 + H 16.29100 14.02800 7.57200 + O 15.18700 13.21600 10.63800 + H 14.57200 12.59700 10.24400 + H 15.59700 12.72600 11.35000 + O 13.97600 9.52900 15.60600 + H 13.09400 9.66700 15.95200 + H 14.43000 10.35500 15.77100 + O 15.34700 7.76500 0.90300 + H 16.07400 8.38700 0.93100 + H 15.52500 7.15400 1.61800 + O 15.71000 5.94700 2.99400 + H 14.83600 5.59400 3.16400 + H 16.06800 6.13100 3.86200 + O 16.11100 8.15800 7.26400 + H 16.50200 7.63300 6.56500 + H 16.47600 9.03400 7.14000 + O 13.24700 11.50700 9.75600 + H 12.62400 12.03200 10.25800 + H 12.87000 11.46000 8.87800 + O 15.51700 6.02000 -1.27900 + H 15.01900 6.26200 -2.06000 + H 15.31200 6.70400 -0.64200 + O 12.94100 2.66900 -0.94700 + H 12.66000 3.45000 -1.42400 + H 12.81200 1.95000 -1.56600 + O 13.10800 4.83300 3.22700 + H 12.99100 4.00200 2.76700 + H 12.76700 4.67000 4.10700 + O -2.83600 5.44200 7.19300 + H -2.68300 5.75100 6.30000 + H -1.97000 5.44700 7.60100 + O 14.01500 6.47700 13.52400 + H 14.74500 5.95400 13.85400 + H 14.43000 7.18200 13.02700 + O 15.10300 2.79900 13.24900 + H 14.28100 2.66200 13.72000 + H 15.72900 2.22200 13.68600 + O 12.67000 0.59500 -2.78700 + H 13.25400 -0.07000 -2.42300 + H 11.82100 0.15700 -2.85800 + O 12.55400 2.49100 1.78300 + H 12.75200 1.59600 2.06000 + H 12.79500 2.51400 0.85700 + O 12.66000 2.36300 7.48000 + H 13.59900 2.29900 7.30100 + H 12.60800 2.56800 8.41400 + O 12.92300 3.08300 10.14300 + H 12.24600 3.27000 10.79300 + H 13.38300 3.91400 10.02900 + O -2.50700 1.37000 13.87400 + H -1.85100 1.12600 14.52800 + H -2.95700 0.55100 13.66800 + O 12.82300 -0.21800 2.50400 + H 13.40200 -0.73300 3.06600 + H 12.50600 -0.84300 1.85100 + O -1.16800 -0.16200 6.34900 + H -0.77800 -1.03200 6.43900 + H -0.42300 0.43800 6.35500 + O -3.33100 2.75800 6.67200 + H -2.67400 2.66700 5.98100 + H -3.19800 3.64300 7.01400 + O 15.97700 -1.61300 13.17500 + H 15.14100 -1.36200 13.56700 + H 15.73900 -2.19700 12.45500 + O 14.97600 -0.89900 -1.84700 + H 15.33800 -1.39500 -2.58100 + H 15.60300 -0.18900 -1.70600 + O -2.22500 0.47700 0.79400 + H -3.02300 0.27900 1.28500 + H -2.53300 0.92600 0.00700 + O 11.57500 14.12400 3.20500 + H 12.42400 14.01100 2.77700 + H 11.72200 14.81500 3.85100 + O -2.33000 -1.24900 10.28600 + H -2.99100 -1.87100 10.58900 + H -2.70800 -0.86500 9.49400 + O -0.50900 11.92000 13.29600 + H -1.44800 11.94500 13.11100 + H -0.44100 12.13400 14.22600 + O 3.07000 12.43400 10.07800 + H 2.75900 13.27700 9.74900 + H 4.01200 12.44200 9.90500 + O 6.48800 13.48700 13.03600 + H 6.53700 13.09000 13.90600 + H 7.24500 14.07000 12.99400 + O 8.67100 15.14100 12.80800 + H 9.08200 14.86100 11.99000 + H 9.40500 15.35000 13.38600 + O 14.93800 -3.48900 11.25600 + H 14.36300 -3.18200 10.55500 + H 15.09300 -4.41100 11.05200 + O 0.10000 14.46500 10.39700 + H -0.66000 14.84300 9.95300 + H 0.11100 14.88900 11.25500 + O 2.52500 14.96800 9.16700 + H 1.63000 14.91700 9.50100 + H 2.43500 15.35000 8.29400 + O 6.96500 14.92600 8.98200 + H 6.45700 15.13600 8.19800 + H 6.89200 15.70700 9.53100 + O 9.36100 14.27500 10.20800 + H 8.58900 14.53000 9.70200 + H 10.08300 14.75500 9.80300 + O 11.85400 15.19100 9.40100 + H 12.31700 16.02800 9.38100 + H 12.09000 14.76700 8.57600 + O -0.85700 12.52100 6.85500 + H -0.83400 12.21200 7.76000 + H 0.05600 12.49700 6.56900 + O 1.83600 12.26500 6.24300 + H 1.82600 11.30900 6.20100 + H 2.72800 12.48500 6.50900 + O 4.54700 12.55700 6.93200 + H 5.00600 11.91100 6.39500 + H 4.92300 12.45400 7.80600 + O 6.15000 -0.86800 6.35200 + H 5.59200 -1.54900 6.72900 + H 6.87500 -1.34700 5.95200 + O 12.79100 -1.99100 6.56800 + H 13.30500 -1.83500 5.77500 + H 12.12900 -1.29900 6.56400 + O 14.38900 -1.78400 4.24000 + H 14.96900 -2.54300 4.18600 + H 14.97500 -1.04600 4.41100 + O -0.38500 -3.70600 3.85500 + H -1.25600 -4.05600 3.66900 + H 0.21300 -4.41600 3.62400 + O -0.27000 -1.47200 0.17600 + H 0.18600 -1.25500 -0.63700 + H -0.96700 -0.81900 0.24200 + O 4.51900 -3.24700 4.25300 + H 5.17000 -2.67000 3.85300 + H 4.68100 -3.17600 5.19400 + O 8.14400 -2.27000 5.20300 + H 8.59000 -1.99700 4.40100 + H 8.49400 -3.14100 5.38800 + O 11.71300 -1.92700 0.59100 + H 11.09500 -1.22700 0.38200 + H 11.16800 -2.70700 0.69100 + O -1.11200 15.14400 0.19700 + H -1.94000 14.73100 -0.05000 + H -1.14400 16.00900 -0.21100 + O 1.28500 -0.88600 -2.11400 + H 1.65800 -1.57300 -2.66800 + H 2.02900 -0.55300 -1.61200 + O 6.01900 -0.19200 0.41600 + H 6.04600 -0.82500 1.13400 + H 6.02400 0.66100 0.85000 + O 7.37300 13.39100 -1.51900 + H 8.25100 13.72600 -1.69700 + H 7.10200 12.97700 -2.33800 + O 13.91300 13.88400 1.67700 + H 13.85700 13.00700 1.29700 + H 13.79600 14.47500 0.93300 + O -0.21000 -1.73100 14.27100 + H -0.71400 -2.53100 14.41800 + H 0.63600 -1.90000 14.68500 + O 1.92000 -2.60900 15.73900 + H 2.65600 -2.88100 15.19000 + H 1.60500 -3.42000 16.13700 + O 0.97100 14.32900 -2.41300 + H 0.78500 14.09200 -1.50400 + H 1.54900 13.63500 -2.72800 + O 10.74600 -3.57000 14.82900 + H 10.89600 -4.17800 15.55300 + H 11.49400 -3.70200 14.24600 + O 13.13000 12.50000 13.67800 + H 12.67000 12.65500 12.85300 + H 13.40400 13.37100 13.96700 + O 18.74700 5.62000 9.28900 + H 18.78500 4.89900 9.91800 + H 19.63200 5.67100 8.92900 + O 20.10700 9.79900 7.47400 + H 20.12600 8.91900 7.09800 + H 20.60400 9.72200 8.28900 + O 17.15300 3.80200 1.84800 + H 16.72200 4.56800 2.22800 + H 16.62500 3.58600 1.08000 + O 17.47300 6.32800 5.58800 + H 17.76700 5.43300 5.41700 + H 18.27700 6.81300 5.77400 + O 20.63700 0.18100 5.26700 + H 21.43900 0.45200 4.82100 + H 20.01000 0.02800 4.56000 + O 18.53900 3.83200 5.50000 + H 19.03700 3.33000 6.14500 + H 18.79100 3.45500 4.65600 + O 20.51200 1.23500 9.74100 + H 20.34900 1.56900 8.85900 + H 19.64300 1.02500 10.08300 + O 20.14200 2.12900 7.09000 + H 20.93800 2.66100 7.11100 + H 20.34100 1.42400 6.47400 + O 21.24700 10.12900 10.25900 + H 21.42500 10.98100 10.65900 + H 20.32500 9.96000 10.45500 + O 17.49500 9.54600 0.79000 + H 17.46300 9.96600 -0.06900 + H 17.74500 10.24800 1.39100 + O 21.63900 7.04800 4.27400 + H 21.63100 6.90400 3.32700 + H 22.56800 7.09100 4.50200 + O 21.41400 5.51100 8.36500 + H 21.89600 4.82200 7.90800 + H 22.03800 6.23300 8.43700 + O 18.18100 6.22900 12.91400 + H 18.10500 7.17000 12.75300 + H 17.51400 5.83500 12.35300 + O 20.97500 11.89300 13.74800 + H 21.38100 12.18100 12.93000 + H 20.03900 12.04800 13.61900 + O 21.32800 6.22700 1.54500 + H 20.56200 6.61100 1.11700 + H 21.06800 5.32700 1.74000 + O 17.05100 7.75800 9.88600 + H 16.63600 7.90000 9.03600 + H 17.64300 7.01900 9.74400 + O 17.46200 10.56100 7.22100 + H 18.37800 10.28200 7.21700 + H 17.39800 11.16300 7.96300 + O 20.04200 13.42100 3.75500 + H 20.18900 12.92600 4.56200 + H 19.68700 12.77800 3.14100 + O 18.58500 9.84100 10.88300 + H 18.56100 9.65500 11.82200 + H 18.02700 9.16900 10.49100 + O 20.96100 -1.55400 7.56400 + H 20.02800 -1.72800 7.44100 + H 21.20100 -0.99900 6.82200 + O 21.30500 4.49900 11.05700 + H 20.53600 3.93000 11.05400 + H 21.40800 4.76800 10.14500 + O 21.89400 3.37300 2.86300 + H 20.95200 3.20900 2.91300 + H 22.29000 2.64900 3.34900 + O 18.67600 3.48800 11.04200 + H 17.85300 3.84300 11.37800 + H 18.53100 2.54300 10.99800 + O 16.31300 0.08300 4.62600 + H 16.61200 0.41500 5.47200 + H 17.07900 0.15400 4.05600 + O 16.81800 1.19000 -1.24200 + H 17.65200 1.56100 -1.53100 + H 16.30900 1.94400 -0.94500 + O 19.24400 2.60200 3.11800 + H 18.54100 3.00100 2.60600 + H 19.07900 1.66100 3.06000 + O 21.64200 2.08600 -1.40100 + H 22.07000 2.67100 -2.02600 + H 20.71900 2.33600 -1.43200 + O 18.57700 9.04100 13.55900 + H 19.34500 8.67000 13.99300 + H 17.85000 8.84400 14.15000 + O 18.72600 -0.19400 3.19000 + H 18.70500 -1.09200 3.52200 + H 18.39200 -0.26100 2.29600 + O 20.00700 7.22200 6.43400 + H 20.53000 7.20900 5.63300 + H 20.42900 6.57900 7.00500 + O 16.26200 4.72900 11.70300 + H 15.81200 4.13200 12.30000 + H 15.59500 4.97300 11.06100 + O 18.21000 5.46900 -1.37800 + H 18.65600 6.11400 -0.82800 + H 17.27900 5.64700 -1.25000 + O 17.91200 0.78000 10.66900 + H 17.01400 1.11000 10.68800 + H 17.81700 -0.17300 10.67700 + O 18.17800 12.24800 9.47500 + H 18.29800 11.45900 10.00500 + H 18.42800 12.96700 10.05500 + O 19.31900 7.53900 0.11400 + H 18.67400 8.10300 0.54200 + H 19.81200 8.12900 -0.45600 + O -21.23300 -0.38300 -1.15500 + H -21.54500 0.52100 -1.19100 + H -20.30100 -0.32400 -1.36100 + O 20.41600 9.40900 -1.65700 + H 20.95600 10.19700 -1.59700 + H 20.79900 8.91000 -2.37900 + O 18.98400 2.85500 -1.84200 + H 18.72800 3.72100 -1.52400 + H 19.09600 2.96900 -2.78500 + O 18.31800 -1.99200 6.87900 + H 18.34100 -2.17800 5.94000 + H 18.19400 -2.84700 7.29000 + O 18.10500 11.86200 2.25800 + H 17.61200 12.05500 3.05600 + H 17.80400 12.51600 1.62700 + O 17.89700 14.03200 15.20000 + H 18.43800 13.81000 15.95800 + H 17.82800 13.21400 14.70600 + O 21.32700 11.10300 16.38900 + H 22.19000 10.72400 16.56100 + H 21.35500 11.35200 15.46500 + O 16.45100 8.39800 15.17800 + H 16.31700 7.45200 15.12000 + H 15.56800 8.76800 15.15700 + O -22.10800 8.28700 -3.75100 + H -21.47500 7.74500 -3.28100 + H -21.64200 8.57500 -4.53600 + O 16.27800 5.65800 14.88300 + H 17.05300 5.58400 14.32600 + H 16.56900 5.35100 15.74200 + O 20.56100 5.16900 13.66700 + H 19.68400 5.47200 13.42900 + H 20.95600 4.90300 12.83700 + O 20.36300 3.19900 15.62000 + H 20.16200 2.44600 15.06400 + H 20.49800 3.92100 15.00600 + O 19.79900 0.80400 14.21900 + H 20.24100 0.54800 13.41000 + H 20.36000 0.46500 14.91700 + O 17.82300 10.19300 -2.01800 + H 18.73200 9.99100 -1.79500 + H 17.54200 9.45800 -2.56400 + O 16.06300 14.49200 3.31500 + H 15.36400 14.23700 2.71400 + H 16.22000 13.71100 3.84700 + O 16.97800 12.35500 4.84900 + H 16.78600 11.51800 5.27000 + H 17.38100 12.88400 5.53900 + O 16.48600 4.95000 7.85700 + H 16.63900 5.39300 7.02200 + H 17.28500 5.10400 8.36000 + O 16.98400 0.94500 13.92500 + H 17.92500 0.94100 14.10300 + H 16.79700 0.06400 13.60100 + O 17.58500 0.68500 7.05000 + H 17.87900 -0.22500 7.06500 + H 18.38300 1.19800 7.17800 + O 15.41300 2.45200 7.33500 + H 16.19100 1.93900 7.11500 + H 15.74600 3.33200 7.50800 + O 17.35000 -0.50800 0.83500 + H 16.49200 -0.90800 0.97800 + H 17.19500 0.15900 0.16600 + O 17.13500 -2.00500 10.22800 + H 16.50100 -2.65200 10.53600 + H 16.65800 -1.49400 9.57400 + O 18.11500 11.78700 13.59400 + H 17.21800 11.75300 13.26100 + H 18.37200 10.86900 13.68300 + O -21.85300 12.57300 11.20300 + H -22.17900 13.29000 10.65900 + H -20.90000 12.66200 11.17200 + O 18.99500 14.51000 10.91200 + H 18.27300 15.12300 10.77100 + H 19.31400 14.71300 11.79200 + O 21.37200 14.62800 9.49700 + H 20.47800 14.67500 9.83700 + H 21.29400 14.89500 8.58100 + O 18.24700 13.60100 7.07200 + H 18.19000 13.21400 7.94600 + H 19.06700 13.26100 6.71300 + O 20.50700 12.24900 6.21900 + H 20.33800 11.37100 6.56000 + H 21.43300 12.40300 6.40300 + O 18.41200 -2.72100 4.20200 + H 17.54000 -3.08000 4.03900 + H 19.00900 -3.44100 3.99900 + O 18.59600 -2.17900 -1.05100 + H 19.21400 -1.70500 -1.60800 + H 18.34500 -1.54600 -0.37800 + O 17.38700 13.59100 0.18700 + H 16.55500 13.53900 -0.28300 + H 17.55000 14.52900 0.28500 + O 20.49200 -1.03700 -2.74500 + H 20.83200 -1.70600 -3.34000 + H 21.22700 -0.83400 -2.16700 + O 19.26200 13.33000 -1.90800 + H 18.72700 13.38800 -1.11700 + H 19.88900 12.63000 -1.72300 + O -2.65800 18.14800 8.78400 + H -2.38800 18.78800 9.44200 + H -2.07100 18.30300 8.04400 + O -8.06500 21.81900 6.53800 + H -8.53600 21.08200 6.92700 + H -8.63300 22.12000 5.82900 + O -15.00000 19.97700 4.93600 + H -15.07500 20.74500 4.37000 + H -15.59800 19.33700 4.55200 + O -8.15200 19.52200 4.00500 + H -7.76600 19.11300 4.78000 + H -7.40900 19.68100 3.42300 + O -1.64300 19.89500 10.78000 + H -1.62600 20.77500 10.40500 + H -1.89400 20.02800 11.69400 + O -18.20200 19.50800 9.93000 + H -18.79100 19.75000 9.21500 + H -18.78400 19.29200 10.65800 + O -6.73100 18.50300 6.59200 + H -7.54600 18.85100 6.95500 + H -6.19500 19.27800 6.42100 + O -15.95500 20.08500 7.47900 + H -15.94500 21.01900 7.68800 + H -15.49400 20.02100 6.64300 + O -5.34900 17.20700 8.83100 + H -5.82000 17.60800 8.10000 + H -4.46400 17.56700 8.77300 + O -13.51000 16.22500 7.01700 + H -13.65100 15.27900 7.05300 + H -14.33600 16.60200 7.32200 + O -4.76900 15.90700 1.13500 + H -5.69200 16.14200 1.04000 + H -4.35600 16.69600 1.48700 + O -16.00100 17.29000 7.76200 + H -16.87700 17.11000 7.41900 + H -15.94300 18.24500 7.78200 + O -11.61100 17.54100 11.25400 + H -11.20800 17.00900 11.94000 + H -12.54500 17.53600 11.46300 + O -8.44000 20.73100 11.88500 + H -9.08500 20.49400 12.55200 + H -8.50000 21.68400 11.82100 + O -11.73700 18.94400 -1.79000 + H -12.16000 18.99700 -0.93300 + H -12.14800 18.19100 -2.21400 + O -15.11100 19.59400 -2.99900 + H -15.34000 19.62600 -2.07000 + H -15.85500 20.00000 -3.44400 + O -19.69800 19.14700 4.28600 + H -19.40500 19.34200 5.17600 + H -19.55600 18.20600 4.18700 + O -20.17800 19.63700 -0.91100 + H -19.36600 19.17000 -1.10600 + H -19.96600 20.55900 -1.05700 + O -12.35800 19.36300 4.48800 + H -13.22600 19.58000 4.82700 + H -11.89500 18.98600 5.23600 + O -18.26000 21.25700 3.16400 + H -18.75200 20.46400 3.37600 + H -17.48000 20.94000 2.70700 + O -15.51900 21.04300 2.01300 + H -14.60500 20.77900 1.90600 + H -15.47800 21.83100 2.55500 + O -9.23500 19.40600 7.33500 + H -9.41600 19.38400 8.27400 + H -9.93600 18.88600 6.94300 + O -16.58200 17.90600 4.09400 + H -17.37600 17.37700 4.02100 + H -15.86700 17.26900 4.06900 + O -13.56300 21.49700 9.12700 + H -14.05300 22.22600 9.50800 + H -12.89700 21.28800 9.78300 + O -9.24900 20.12900 -1.99600 + H -10.11000 19.73200 -1.86900 + H -8.85300 20.12900 -1.12400 + O -8.32800 20.28900 0.61700 + H -7.50100 20.01600 1.01400 + H -8.51400 21.14100 1.01200 + O -20.03600 18.74900 11.87700 + H -20.60100 18.70200 12.64800 + H -20.33500 18.03400 11.31600 + O -9.82400 19.21100 9.98400 + H -9.19700 19.51100 10.64200 + H -10.37000 18.57400 10.44500 + O -9.58100 18.26200 2.00000 + H -9.35800 18.91800 1.33900 + H -9.20100 18.60100 2.81100 + O -12.63200 20.84900 2.19900 + H -12.28000 21.73700 2.26000 + H -12.38700 20.43600 3.02600 + O -11.54300 21.13400 11.02400 + H -11.39300 20.81500 11.91300 + H -11.00200 20.57000 10.47200 + O -11.97600 17.23200 2.78900 + H -12.13600 17.97100 3.37600 + H -11.12200 17.41700 2.39800 + O -15.72400 19.41900 -0.31000 + H -15.91600 20.02800 0.40300 + H -14.88700 19.02200 -0.07000 + O -5.75700 20.26300 11.38600 + H -6.69100 20.29800 11.59200 + H -5.71400 20.40600 10.44100 + O -20.54300 -20.95600 -4.09200 + H -20.26800 -21.24900 -3.22300 + H -20.49100 -21.74000 -4.63800 + O -18.64500 16.80000 7.06400 + H -18.91000 16.63000 6.16000 + H -18.92200 16.02000 7.54500 + O -15.03700 21.62700 12.14900 + H -15.13800 20.81900 11.64600 + H -15.09300 22.32500 11.49600 + O -15.49400 19.55000 10.39500 + H -15.13200 19.86700 9.56800 + H -16.43800 19.50500 10.24300 + O -16.51500 21.67600 14.42500 + H -17.29500 21.15200 14.24600 + H -16.00800 21.63700 13.61400 + O -14.37300 17.25500 11.47000 + H -14.69700 18.13100 11.26300 + H -14.94000 16.66500 10.97300 + O -12.89200 20.08500 15.01900 + H -12.58200 19.66100 15.81900 + H -13.79100 19.77200 14.91500 + O -10.54100 20.11300 13.54600 + H -11.38500 20.09500 13.99800 + H -9.93300 19.69700 14.15700 + O -8.79400 18.87600 15.33700 + H -8.77000 17.93400 15.17100 + H -9.15000 18.95500 16.22200 + O -18.56100 19.91800 13.98400 + H -18.91700 19.54500 13.17800 + H -18.33000 19.15800 14.51900 + O -14.71800 15.32200 13.46100 + H -13.88900 14.86300 13.32500 + H -14.68300 16.06500 12.85900 + O -14.16600 16.81700 15.77500 + H -14.41200 16.23100 15.05900 + H -14.69000 17.60500 15.63200 + O -6.75600 20.57500 -4.79200 + H -5.96500 20.18500 -5.16300 + H -7.41200 19.88000 -4.84200 + O -6.06500 19.91600 2.19900 + H -5.49800 19.15200 2.09000 + H -5.46200 20.64400 2.34600 + O -5.58300 21.22400 8.83800 + H -5.42900 20.99100 7.92300 + H -5.64600 22.17900 8.83500 + O -21.16800 18.28900 14.40300 + H -20.76800 18.63700 15.20000 + H -20.71700 17.45900 14.25000 + O -3.42200 16.79800 14.82300 + H -2.82400 16.06400 14.68400 + H -3.31400 17.34600 14.04500 + O -18.62400 19.59500 6.85000 + H -18.76600 18.67400 7.06700 + H -17.71900 19.76700 7.11000 + O -4.08200 18.50400 12.65400 + H -4.64700 19.18600 12.29200 + H -4.37300 17.69900 12.22700 + O -22.08900 17.79200 0.79200 + H -22.49500 18.26600 0.06600 + H -21.21800 18.18000 0.87400 + O -20.87000 16.75200 10.13800 + H -21.38300 15.94900 10.04700 + H -21.29300 17.37500 9.54600 + O -5.09300 16.27700 11.46800 + H -5.29000 16.62400 10.59800 + H -4.77000 15.39100 11.30500 + O -11.21900 17.69500 6.34800 + H -11.99800 17.23100 6.65500 + H -10.69200 17.02100 5.91900 + O -7.10300 15.91600 5.58900 + H -6.87900 16.79700 5.88600 + H -8.05300 15.93700 5.47100 + O -19.23500 16.39300 4.35400 + H -20.12100 16.03700 4.28800 + H -18.66300 15.63200 4.25300 + O -21.78400 15.24100 -0.32500 + H -22.14500 15.47500 -1.18000 + H -21.89800 16.02900 0.20600 + O -14.24500 16.33200 4.21500 + H -13.48900 16.56800 3.67800 + H -13.91000 16.32300 5.11100 + O -9.88900 15.63500 5.09900 + H -10.15700 15.27100 4.25500 + H -9.87900 14.88400 5.69200 + O -7.43300 16.66500 1.09800 + H -8.14900 17.24100 1.36500 + H -7.80700 15.78400 1.13700 + O -17.85200 18.29000 -1.70500 + H -17.52800 17.45500 -2.04200 + H -17.14400 18.61600 -1.14900 + O -13.18400 18.74400 0.55200 + H -13.00400 17.94300 1.04400 + H -12.89800 19.45000 1.13200 + O -19.34800 16.21500 13.97700 + H -19.60400 15.42900 14.46000 + H -18.73800 16.66600 14.56100 + O -8.56500 16.13700 14.97400 + H -8.66200 15.71800 15.82900 + H -7.64000 16.02600 14.75600 + O 16.06200 18.49000 9.16500 + H 16.45000 19.13200 9.76100 + H 16.55100 18.59500 8.34900 + O 9.93600 21.60400 6.77100 + H 9.52300 20.75900 6.94800 + H 9.53000 21.90300 5.95700 + O 3.28800 19.96300 4.39900 + H 3.57500 20.82400 4.09500 + H 2.75800 19.62000 3.67900 + O 10.66900 20.11200 3.69000 + H 10.94900 19.57600 4.43200 + H 11.47000 20.26700 3.19000 + O 17.55700 20.06800 10.91800 + H 17.63300 21.01500 10.79500 + H 17.31700 19.96400 11.83800 + O 1.50100 18.24000 8.51200 + H 0.76600 18.47500 7.94600 + H 1.09800 18.03200 9.35500 + O 11.36800 18.63500 5.93200 + H 10.60200 18.68700 6.50500 + H 11.96200 19.30800 6.26500 + O 1.27400 20.62500 6.21200 + H 1.77000 21.16400 6.82900 + H 1.93400 20.26900 5.61900 + O 13.49100 17.43700 9.46900 + H 12.92700 18.20000 9.34500 + H 14.37100 17.75300 9.26300 + O 5.12700 15.31100 6.78900 + H 4.92400 14.37500 6.79400 + H 4.27300 15.74200 6.81000 + O 13.36500 15.70800 -0.42000 + H 12.55000 15.94800 0.02000 + H 13.61000 16.49000 -0.91500 + O 2.59800 16.43200 6.78000 + H 2.03400 16.50100 6.00900 + H 2.31800 17.15100 7.34700 + O 7.16600 16.78300 11.10000 + H 7.49700 16.29000 11.85200 + H 6.21500 16.69400 11.15500 + O 10.41100 -21.91800 9.43900 + H 10.20600 -21.96600 8.50500 + H 9.97900 -21.11800 9.73900 + O 6.91500 18.16200 -1.47900 + H 6.49900 18.37300 -0.64300 + H 6.34900 17.49500 -1.86700 + O 3.96200 19.31900 -2.84000 + H 3.64400 19.47500 -1.95000 + H 3.35500 19.80300 -3.40000 + O 5.97500 19.06600 4.52100 + H 5.05200 19.26400 4.68100 + H 6.27800 18.66000 5.33300 + O -0.38200 20.96400 1.86800 + H -1.09600 20.34600 1.71500 + H 0.37600 20.41300 2.06300 + O 4.04500 21.57500 1.22300 + H 4.94300 21.33000 1.44700 + H 3.72600 22.04600 1.99300 + O 9.05500 19.08300 7.44700 + H 9.07300 18.95800 8.39500 + H 8.31900 18.54900 7.14800 + O 1.72700 19.12000 2.26900 + H 1.12600 18.37500 2.27600 + H 2.14500 19.08400 1.40900 + O 5.52700 20.84700 8.80800 + H 5.05300 21.59000 9.18200 + H 6.20100 20.64200 9.45600 + O 9.47200 18.84900 -1.94100 + H 8.59600 18.51300 -1.74900 + H 9.83600 19.07400 -1.08500 + O 10.40800 20.26600 0.25700 + H 11.17200 20.40300 0.81800 + H 10.53400 20.87200 -0.47300 + O 0.18400 17.76700 10.82100 + H -0.47700 18.45400 10.90000 + H 0.01600 17.18500 11.56200 + O 8.96300 18.65200 10.19800 + H 9.56600 18.80300 10.92600 + H 8.41000 17.92700 10.49000 + O 9.19300 18.46500 2.01100 + H 9.30600 19.07900 1.28600 + H 9.54500 18.92500 2.77300 + O 6.49000 20.76400 2.37500 + H 6.85900 21.60800 2.63400 + H 6.41000 20.27500 3.19400 + O 7.09800 20.49100 11.09800 + H 6.82600 20.06200 11.91000 + H 7.79000 19.92800 10.74900 + O 6.92700 17.06100 2.86600 + H 6.62500 17.70400 3.50700 + H 7.76300 17.40700 2.55400 + O 3.16700 19.40400 -0.16500 + H 3.29800 20.28600 0.18400 + H 3.94100 18.91700 0.11700 + O 12.86700 19.81200 11.77800 + H 11.99800 19.72000 12.16800 + H 12.71100 19.77300 10.83400 + O -0.55800 21.13800 -2.32000 + H -1.08100 21.61800 -1.67800 + H -0.22400 20.38100 -1.84000 + O 0.53600 16.72600 4.84000 + H 0.25900 17.05500 3.98400 + H 0.65200 15.78500 4.70900 + O 3.22700 20.86800 11.57000 + H 3.41100 20.08700 11.04900 + H 3.44500 21.59900 10.99200 + O 3.85100 18.86200 9.74900 + H 4.41100 19.42000 9.21000 + H 3.06600 18.72600 9.21900 + O 1.58700 20.39200 13.72800 + H 0.85700 19.78800 13.59100 + H 1.92300 20.56600 12.84900 + O 4.44900 16.33800 10.62200 + H 4.24900 17.24500 10.39300 + H 3.84500 15.81700 10.09300 + O 6.27800 19.60100 13.67900 + H 6.79800 19.27300 14.41400 + H 5.37900 19.35200 13.89100 + O 10.23000 19.47400 12.58900 + H 9.77500 20.27700 12.84100 + H 10.44800 19.05000 13.41900 + O 10.92600 18.35700 14.99200 + H 10.92800 17.41700 14.81200 + H 10.37200 18.45200 15.76700 + O -0.34100 18.37400 14.09900 + H -0.28800 17.52900 13.65200 + H 0.00200 18.20300 14.97600 + O 4.23800 15.07500 13.02800 + H 4.91000 14.39600 12.96500 + H 4.24200 15.49600 12.16800 + O 4.97300 16.65800 15.11900 + H 4.78100 16.05700 14.39900 + H 4.48000 17.45100 14.90600 + O 12.43300 19.48400 -3.64000 + H 12.74300 18.90400 -2.94500 + H 11.68300 19.02500 -4.01900 + O 12.71700 20.65900 1.77600 + H 13.09600 19.78400 1.85700 + H 13.28200 21.10900 1.14800 + O 12.17700 19.88900 9.16300 + H 12.51500 20.19600 8.32100 + H 11.47800 20.50500 9.38100 + O -2.21900 20.28500 13.52900 + H -1.54100 19.69400 13.85600 + H -2.97200 19.71700 13.36300 + O 16.08000 16.13600 15.22900 + H 16.62200 15.35100 15.31400 + H 16.63600 16.76200 14.76500 + O 13.51200 18.04800 1.92000 + H 13.80000 17.56300 2.69300 + H 12.77000 17.54600 1.58400 + O -0.61000 18.57000 6.75000 + H -0.40200 17.91400 6.08500 + H -0.18900 19.37100 6.43700 + O 14.78800 18.16400 12.83700 + H 14.08800 18.73300 12.51700 + H 14.61800 17.31800 12.42300 + O 14.00700 17.96200 -1.89400 + H 14.58100 17.51900 -2.51800 + H 14.50600 18.72800 -1.61100 + O -2.22600 15.47400 9.35800 + H -2.87300 15.17700 9.99700 + H -2.40900 16.40700 9.24600 + O 14.40500 15.73300 11.49400 + H 13.98400 16.17800 10.75800 + H 14.60300 14.85700 11.16300 + O 7.07700 17.31900 6.42300 + H 6.39000 16.67200 6.58200 + H 7.85800 16.79900 6.23200 + O 12.11500 16.08400 5.19200 + H 12.97900 16.25000 4.81500 + H 11.81000 16.94400 5.48100 + O 14.52300 16.66800 4.07000 + H 15.13100 15.96000 3.85500 + H 15.07900 17.36700 4.41200 + O -0.22500 17.17200 2.17400 + H -0.93000 17.77700 1.94500 + H -0.44500 16.36200 1.71300 + O -1.67000 17.63800 -0.91700 + H -0.86000 18.09200 -1.14900 + H -1.99000 18.10000 -0.14100 + O 5.03600 15.10500 3.94200 + H 5.60600 15.75800 3.53600 + H 5.18800 15.20500 4.88200 + O 9.28900 15.75900 6.04100 + H 10.18600 15.79900 5.71000 + H 9.17700 14.85000 6.32100 + O 11.20300 16.71800 1.07700 + H 10.52700 17.32300 1.38300 + H 10.75400 15.87700 0.99500 + O 0.81200 18.74600 -1.58700 + H 1.14300 18.12200 -2.23300 + H 1.57200 18.95300 -1.04300 + O 5.61100 18.59400 0.88800 + H 5.84600 17.87100 1.47000 + H 5.94000 19.37600 1.33100 + O -0.04200 15.85600 12.92500 + H -0.75400 15.36300 13.33300 + H 0.73000 15.64100 13.44800 + O 2.04400 15.23800 14.67100 + H 2.79600 15.23300 14.07800 + H 2.07400 14.38500 15.10400 + O 10.86600 15.61400 14.53800 + H 10.61500 15.18100 15.35400 + H 11.79600 15.41300 14.43600 + O 21.43500 19.62300 4.61800 + H 22.34900 19.49800 4.36300 + H 20.93600 19.11400 3.98000 + O 21.36600 18.26000 8.52400 + H 21.10100 19.01700 8.00100 + H 20.68300 18.17800 9.19000 + O 20.18600 20.24900 6.94800 + H 20.04700 21.19600 6.94200 + H 20.63200 20.06700 6.12100 + O 21.22600 15.95700 6.99300 + H 20.32100 16.09700 6.71500 + H 21.46600 16.76700 7.44500 + O 16.19900 18.66000 4.82200 + H 16.66100 18.78100 5.65100 + H 16.71900 19.14800 4.18400 + O 15.37900 20.23100 -1.03600 + H 16.16000 20.63600 -1.41200 + H 14.85500 20.96700 -0.72000 + O 17.72400 19.76900 2.74500 + H 17.22700 19.28200 2.08700 + H 18.56200 19.31000 2.80200 + O 19.93800 18.07900 2.82700 + H 19.48500 17.26000 3.02600 + H 20.55300 17.84900 2.13000 + O 19.31800 18.02700 10.29900 + H 18.84000 18.83500 10.48500 + H 18.70100 17.32800 10.51900 + O 21.13500 19.15100 -1.50100 + H 20.78700 19.84600 -2.05900 + H 22.06000 19.37100 -1.39100 + O 17.51400 -22.18200 -1.97200 + H 17.73500 -21.56400 -1.27500 + H 17.16300 -21.63600 -2.67500 + O 18.61900 16.23900 6.21500 + H 18.52900 16.19500 5.26300 + H 18.32000 15.38400 6.52500 + O 20.44800 19.82000 14.54200 + H 19.70000 19.25600 14.34700 + H 21.21400 19.28100 14.34200 + O 18.46800 17.71900 14.44000 + H 18.86200 17.04000 13.89200 + H 18.73800 17.49400 15.33000 + O 16.81000 19.78800 13.56400 + H 17.43100 19.18300 13.97000 + H 16.04400 19.24900 13.36700 + O 17.83700 18.74700 7.06200 + H 18.17200 17.87800 6.84000 + H 18.61600 19.30100 7.10100 + O 16.10700 18.44000 0.91300 + H 15.25500 18.20600 1.27900 + H 15.90500 19.05000 0.20400 + O 17.03800 16.39300 10.66500 + H 16.27400 16.14300 11.18400 + H 16.70600 17.05000 10.05200 + O 18.56900 15.73900 3.49500 + H 17.68900 15.44700 3.25600 + H 19.10100 14.94400 3.46800 + O 17.61300 16.28300 -0.03200 + H 18.14300 16.67800 -0.72500 + H 17.13900 17.01800 0.35700 + O 19.28300 17.23300 -2.07200 + H 19.83700 16.51400 -2.37600 + H 19.90000 17.90300 -1.77800 + O 19.64100 15.32500 13.53100 + H 18.98100 14.85300 14.04000 + H 20.43500 15.27600 14.06400 + O 21.32500 14.99000 15.79000 + H 22.23800 14.70500 15.82100 + H 20.84500 14.30800 16.26100 + O -20.94500 -15.27000 -21.23500 + H -20.65400 -14.37800 -21.04500 + H -20.96300 -15.31800 -22.19100 + O -9.55700 -9.24700 20.70000 + H -9.91200 -9.30500 19.81300 + H -10.07200 -9.87600 21.20700 + O -7.49200 -8.69800 -21.50300 + H -8.06700 -8.76000 -22.26600 + H -7.99700 -9.09200 -20.79100 + O -10.93700 -12.32400 19.67900 + H -11.06300 -11.86800 20.51100 + H -11.67800 -12.04900 19.14000 + O -10.58700 -13.36200 15.96300 + H -9.98100 -13.32500 16.70300 + H -10.27600 -14.09900 15.43600 + O -8.11100 -11.98500 19.88400 + H -9.05700 -11.99300 19.73600 + H -8.01600 -12.05800 20.83300 + O -8.32300 -13.55500 17.60900 + H -8.08400 -12.91000 18.27400 + H -7.86900 -13.26500 16.81800 + O -4.75600 21.37300 18.29500 + H -5.36000 20.94300 17.69000 + H -4.48600 22.17000 17.84000 + O -16.30200 -12.32000 18.88800 + H -16.05100 -12.46500 19.80100 + H -16.04800 -13.12500 18.43800 + O -3.60600 -6.94300 17.43400 + H -3.44500 -6.00400 17.51900 + H -2.73400 -7.33800 17.43900 + O -11.64800 -3.37500 20.54700 + H -11.62900 -3.23900 19.60000 + H -11.66600 -2.49200 20.91600 + O -8.57400 -4.36600 18.15000 + H -7.95100 -5.02000 17.83300 + H -8.55000 -4.45300 19.10300 + O -9.08900 -4.30300 20.95800 + H -9.99100 -4.02200 20.80100 + H -9.08500 -4.60100 21.86700 + O -7.25300 -12.70100 15.27600 + H -7.61000 -12.61800 14.39200 + H -6.44200 -12.19300 15.25700 + O -11.49100 -18.13600 17.38600 + H -12.14400 -18.52000 17.97100 + H -11.80200 -18.34900 16.50600 + O -10.48300 -15.00900 20.33200 + H -10.64000 -14.15100 19.93800 + H -9.64500 -15.29100 19.96700 + O -14.90200 -17.83800 15.77600 + H -14.85900 -17.62700 16.70900 + H -15.33400 -17.08100 15.37900 + O -21.00000 -15.43500 19.95900 + H -20.30900 -15.13500 19.36900 + H -21.80300 -15.06000 19.59900 + O -14.99600 -16.66400 18.29600 + H -14.26200 -16.56200 18.90200 + H -15.19100 -15.77400 18.00300 + O -15.38900 -14.23400 17.07100 + H -15.70800 -14.63600 16.26200 + H -14.63400 -13.71400 16.79900 + O -16.63200 -20.54800 21.18000 + H -16.08800 -21.33200 21.09400 + H -16.55000 -20.10500 20.33500 + O -13.31800 -10.46500 19.51900 + H -13.96000 -10.39400 18.81300 + H -13.81800 -10.29300 20.31700 + O -8.92900 -18.15200 18.25300 + H -9.83200 -18.04900 17.95300 + H -8.63900 -17.26200 18.45400 + O -8.05600 -15.73400 19.23000 + H -7.14400 -15.76500 19.51800 + H -8.08100 -15.02600 18.58800 + O -11.08100 -9.42100 18.33800 + H -11.30000 -8.49100 18.38900 + H -11.84300 -9.86800 18.70600 + O -4.14500 -14.97300 17.55300 + H -4.63600 -15.35600 16.82700 + H -4.22900 -14.02700 17.42800 + O -5.13300 -7.60100 19.64400 + H -5.50400 -8.47100 19.49500 + H -4.64600 -7.40700 18.84300 + O -18.68000 -11.47800 17.76900 + H -17.86200 -11.71000 18.21000 + H -18.90300 -10.61500 18.11700 + O -6.93300 -8.92500 17.06000 + H -6.62000 -9.50500 17.75400 + H -7.75200 -9.32300 16.76500 + O -3.14600 -4.26300 17.83300 + H -3.22400 -3.68700 17.07300 + H -3.81100 -3.94600 18.44400 + O -5.88100 -10.50200 19.06000 + H -6.61500 -11.00200 19.41700 + H -5.16500 -10.65200 19.67800 + O -9.01400 -18.74600 20.92200 + H -8.94500 -18.71900 19.96800 + H -9.04700 -17.82600 21.18700 + O -11.85900 -6.86000 18.87100 + H -12.79000 -6.65100 18.94000 + H -11.49700 -6.63200 19.72700 + O -12.89500 -16.50700 20.12100 + H -12.00100 -16.18300 20.01800 + H -13.29300 -15.91700 20.76100 + O -11.42200 -20.10400 21.18200 + H -11.74900 -19.69400 21.98300 + H -10.52300 -19.78600 21.09900 + O -14.44300 -5.98900 19.05900 + H -14.43800 -5.12100 18.65500 + H -14.99100 -6.52000 18.48100 + O -16.01500 -19.16200 18.83900 + H -15.93400 -18.21400 18.73000 + H -15.11200 -19.47700 18.88100 + O -19.59700 -14.16200 17.94100 + H -19.30300 -13.25100 17.94400 + H -19.80500 -14.34300 17.02400 + O -11.58200 -5.74800 16.36300 + H -11.73600 -6.19400 17.19600 + H -10.82000 -6.19100 15.99100 + O -7.07100 -6.17800 16.70400 + H -6.84100 -7.06700 16.97300 + H -6.53400 -6.01500 15.92800 + O -9.40100 -9.76700 16.18800 + H -10.02800 -9.81800 16.90900 + H -9.78300 -10.31100 15.49900 + O -15.07000 -10.38100 17.32000 + H -15.54500 -11.10000 17.73700 + H -14.73300 -10.75500 16.50600 + O -13.29900 -12.75200 16.13300 + H -13.41300 -12.71000 15.18400 + H -12.37000 -12.94700 16.25500 + O -4.99600 -11.17200 14.89600 + H -4.84000 -11.43900 13.99000 + H -4.79000 -10.23700 14.90900 + O -13.95800 -3.51900 17.96300 + H -13.00700 -3.41500 17.93600 + H -14.29700 -2.75600 17.49300 + O -3.65000 -10.78800 20.65400 + H -2.93600 -10.16600 20.51500 + H -3.21200 -11.61400 20.85900 + O -2.82100 -13.33700 -22.30200 + H -3.53200 -13.91800 -22.03200 + H -2.33100 -13.16800 -21.49800 + O -4.24500 -12.17200 17.36800 + H -4.37900 -11.88300 16.46500 + H -4.80100 -11.59300 17.89000 + O -5.59200 -16.17600 15.53200 + H -5.80800 -15.96900 14.62300 + H -5.60500 -17.13300 15.57000 + O -5.71500 -18.95500 15.43900 + H -5.18400 -19.46200 16.05300 + H -6.61100 -19.25300 15.59500 + O -5.34200 -15.83600 19.90000 + H -4.91300 -16.66600 20.11100 + H -4.89500 -15.53000 19.11100 + O -4.19600 -18.16500 20.91900 + H -3.78300 -18.14900 21.78300 + H -4.98800 -18.68900 21.04100 + O -3.79800 -20.32600 16.94800 + H -3.13000 -20.57200 16.30900 + H -3.35500 -19.71300 17.53400 + O 21.43700 -17.84900 19.67500 + H 21.41800 -17.84800 18.71800 + H 22.01200 -17.12000 19.90500 + O -19.46500 -21.83600 17.53800 + H -19.14600 -20.94200 17.41300 + H -19.49100 -21.95100 18.48800 + O -6.46600 -19.54600 21.68500 + H -7.38200 -19.48600 21.41500 + H -6.21600 -20.44800 21.48700 + O -18.32500 -19.27700 17.12400 + H -17.81700 -19.40400 16.32200 + H -17.66800 -19.23600 17.81900 + O -13.23900 -19.14000 19.33600 + H -12.64900 -19.64700 19.89300 + H -13.20000 -18.25100 19.69000 + O -11.16300 -3.45300 17.80900 + H -10.22700 -3.56300 17.97700 + H -11.37400 -4.13300 17.16900 + O -4.76100 -5.48200 21.35100 + H -4.86200 -6.24300 20.78000 + H -4.76200 -4.73200 20.75600 + O -0.62300 -14.66800 20.49400 + H -1.39700 -14.14700 20.70800 + H -0.89800 -15.21600 19.75800 + O 8.69600 -8.05300 20.56200 + H 8.31300 -7.65500 19.78000 + H 8.48000 -7.44600 21.27000 + O -1.63600 -8.88700 20.07700 + H -1.67000 -8.60600 19.16300 + H -1.18200 -8.17600 20.53000 + O 8.32900 -12.43000 17.76800 + H 7.55200 -11.87800 17.85400 + H 8.45100 -12.52600 16.82300 + O 6.92400 -15.67800 16.03100 + H 6.75900 -16.60600 16.19900 + H 7.87100 -15.58100 16.13100 + O 13.78100 21.43000 18.62000 + H 13.37500 20.79700 18.02700 + H 14.07600 22.13900 18.04900 + O 1.58600 -12.96100 19.95100 + H 1.02900 -12.24400 19.64800 + H 0.97800 -13.67500 20.13800 + O 8.68500 -4.24100 19.92500 + H 8.52100 -4.30900 18.98400 + H 8.40100 -3.35600 20.15400 + O 11.37300 -5.29700 17.00400 + H 11.84100 -6.09400 16.75600 + H 11.46000 -5.25400 17.95600 + O 11.40300 -4.92700 19.79500 + H 10.48200 -4.79400 20.01900 + H 11.80000 -5.26200 20.59900 + O 6.55200 -18.37400 16.72900 + H 6.03000 -18.51300 17.51900 + H 6.14000 -18.93000 16.06800 + O 7.99300 -14.34800 19.67700 + H 8.03100 -13.57200 19.11800 + H 8.69200 -14.91500 19.34900 + O 3.73900 -17.56800 14.50700 + H 3.51500 -17.55800 15.43800 + H 3.11700 -16.96200 14.10400 + O -1.65700 -16.00300 18.25100 + H -1.02000 -15.70400 17.60200 + H -2.48200 -15.60100 17.98000 + O 3.54100 -15.99000 20.46400 + H 4.39800 -16.10400 20.05400 + H 2.98300 -15.65100 19.76400 + O 2.57400 -15.31700 17.89600 + H 1.72500 -15.06600 17.53300 + H 3.21400 -14.92500 17.30200 + O 1.73200 -19.41700 21.61600 + H 1.58300 -20.29700 21.27000 + H 1.40200 -18.83200 20.93500 + O 6.11800 -10.88600 18.46100 + H 5.22500 -11.21300 18.35800 + H 6.19100 -10.67600 19.39200 + O 8.98500 -18.60300 17.97100 + H 8.14800 -18.52500 17.51200 + H 9.24800 -17.70000 18.14600 + O 9.86100 -16.03900 18.63300 + H 10.77500 -16.10200 18.90900 + H 9.90500 -15.85800 17.69400 + O 7.98100 -9.23800 17.21300 + H 7.78100 -8.39300 17.61500 + H 7.28400 -9.81900 17.51800 + O 13.99900 -16.57300 17.53800 + H 13.71900 -17.41600 17.18200 + H 13.69900 -15.93000 16.89600 + O -1.53100 -12.76600 17.74100 + H -1.07700 -12.11900 18.28200 + H -2.46000 -12.57000 17.86100 + O 4.82900 -5.11700 -21.99900 + H 3.89400 -5.30700 -21.92600 + H 5.03000 -4.59600 -21.22100 + O 15.70300 -5.51000 18.23400 + H 15.75500 -6.28700 17.67800 + H 15.16400 -4.89400 17.73700 + O 5.07900 -8.14100 16.05200 + H 5.02500 -9.00000 15.63200 + H 4.72100 -8.28000 16.92900 + O 3.83100 -7.97900 18.55100 + H 3.91700 -8.37500 19.41800 + H 4.11700 -7.07400 18.67100 + O -0.03200 -10.90300 19.12500 + H -0.56500 -10.32100 19.66700 + H 0.54300 -10.31400 18.63600 + O 8.86500 -19.23400 20.64000 + H 8.94900 -19.16400 19.68900 + H 8.99600 -18.34100 20.95900 + O 7.32700 -6.90800 18.53100 + H 6.44800 -6.60000 18.75500 + H 7.71500 -6.18600 18.03600 + O 6.15800 -16.43700 19.83200 + H 6.64300 -15.73700 19.39400 + H 6.46200 -16.40800 20.73900 + O 6.39900 -20.47900 20.91200 + H 6.25800 -20.24800 21.83000 + H 7.30700 -20.22800 20.73900 + O 5.03000 -5.56200 19.18800 + H 5.29500 -4.74400 18.76600 + H 4.91000 -5.32900 20.10800 + O 3.18200 -17.89200 17.22200 + H 2.99100 -17.03400 17.60000 + H 3.77500 -18.30500 17.85000 + O 1.37900 -8.97800 17.70400 + H 2.13400 -8.49000 18.03300 + H 1.73300 -9.52300 17.00200 + O 0.21000 -14.55200 16.60300 + H -0.33100 -13.87700 17.01100 + H 0.26800 -14.29100 15.68400 + O -0.20000 -6.62900 21.08900 + H -0.45300 -5.99900 20.41300 + H 0.68100 -6.36000 21.34900 + O 7.36600 -6.66100 15.44300 + H 6.65300 -7.24900 15.69400 + H 8.09300 -7.24500 15.22400 + O 12.98500 -7.38900 16.17800 + H 13.93100 -7.43600 16.31600 + H 12.88700 -7.21400 15.24200 + O 2.46200 -10.80400 15.98600 + H 2.77000 -11.31100 16.73700 + H 3.24400 -10.67100 15.45000 + O 10.05300 -10.29400 15.75400 + H 9.43600 -9.88400 16.36000 + H 9.55200 -10.99800 15.34100 + O 3.63200 -12.20700 18.08900 + H 3.05500 -12.57800 18.75600 + H 3.96500 -12.96600 17.61100 + O 4.52600 -14.19300 16.30000 + H 4.44100 -13.80300 15.43000 + H 5.35700 -14.66800 16.26900 + O 6.25100 -3.47200 17.87200 + H 7.06000 -3.77500 17.45900 + H 5.89300 -2.83200 17.25700 + O -1.24200 -8.29000 17.34200 + H -0.29900 -8.39100 17.47200 + H -1.40300 -8.64400 16.46700 + O 12.43600 -16.11800 15.10100 + H 12.57600 -16.04800 14.15700 + H 12.62400 -17.03400 15.30500 + O 12.90900 -18.69000 16.08900 + H 13.53500 -19.38300 16.30000 + H 12.07500 -19.14600 15.97800 + O 12.26800 -16.40500 19.77300 + H 12.51500 -16.81300 20.60200 + H 13.05000 -16.47700 19.22600 + O 15.29500 -20.95100 20.59300 + H 14.76100 -21.43300 19.96200 + H 14.77100 -20.94300 21.39400 + O 14.85000 -20.55900 16.93700 + H 15.10900 -21.00900 16.13300 + H 15.67500 -20.25900 17.31900 + O -2.54100 -18.59200 18.73800 + H -3.11400 -18.48000 19.49600 + H -2.21000 -17.71200 18.55600 + O -0.38500 -20.26400 18.43900 + H 0.05200 -19.75000 17.76000 + H -1.16200 -19.75200 18.66500 + O 10.80600 -20.02900 -21.52900 + H 10.11900 -19.86900 -22.17600 + H 11.33500 -20.73000 -21.90900 + O -0.76200 -4.71600 19.19800 + H -1.59000 -4.63300 18.72400 + H -0.52300 -3.81700 19.42200 + O 0.84500 -18.98400 16.35000 + H 1.14200 -19.80800 15.96300 + H 1.64600 -18.55700 16.65100 + O 5.09700 -18.80900 19.02800 + H 5.34400 -19.49100 19.65200 + H 5.45700 -18.00300 19.39900 + O 8.58300 -4.84900 17.11400 + H 9.53100 -4.94000 17.01500 + H 8.21700 -5.34500 16.38200 + O 15.02400 -4.64500 20.75700 + H 15.27000 -5.04000 19.92000 + H 15.28500 -3.72800 20.67600 + O 0.74400 -4.85100 16.79300 + H 0.45000 -5.05900 17.68000 + H 1.40800 -5.51200 16.59900 + O 17.95100 -16.23300 21.44400 + H 18.06500 -15.93400 20.54200 + H 17.33100 -16.96000 21.38000 + O 16.51600 -16.67100 18.71000 + H 17.15700 -16.47300 18.02700 + H 15.66700 -16.57300 18.27900 + O 20.87300 -14.26600 18.66700 + H 20.73700 -14.39000 17.72700 + H 20.73800 -13.32900 18.80700 + O 20.14800 -19.48000 21.47200 + H 19.49600 -19.94800 20.94900 + H 20.58400 -18.90000 20.84800 + O 21.53900 -17.54300 16.85700 + H 21.17200 -16.68900 16.62700 + H 22.40300 -17.55100 16.44600 + O 18.21200 -16.06400 16.62000 + H 19.05800 -15.63700 16.48300 + H 18.14800 -16.70300 15.91000 + O 16.03700 -18.22200 20.90900 + H 15.86200 -19.12700 20.65100 + H 16.16500 -17.75600 20.08300 + O 17.49300 -20.22100 17.81900 + H 18.27600 -19.94400 17.34300 + H 17.78200 -20.31000 18.72700 + O 19.78800 -19.65700 16.42100 + H 20.22000 -20.26700 15.82300 + H 20.46700 -19.02200 16.64700 + O 21.79600 5.05200 21.68700 + H 21.05200 4.97000 21.08900 + H 22.48200 4.51500 21.29000 + O -9.80000 9.94700 19.80300 + H -10.38000 10.08200 19.05400 + H -9.16600 10.66100 19.74700 + O -20.89200 8.86000 19.72100 + H -21.04800 9.11300 18.81100 + H -20.36900 9.57500 20.08400 + O -7.13600 10.92100 20.63500 + H -7.01000 10.14900 20.08300 + H -7.46400 10.57200 21.46400 + O -11.66900 6.24600 18.84400 + H -12.32000 6.90500 19.08600 + H -11.78900 6.12500 17.90200 + O -10.75300 5.17200 16.21500 + H -10.12600 5.27500 16.93100 + H -10.32000 4.57900 15.60100 + O -9.35600 7.25700 19.94800 + H -9.52400 8.19800 20.00000 + H -10.18300 6.88300 19.64400 + O -8.65500 5.31700 18.03700 + H -8.70300 6.00500 18.70100 + H -8.19700 5.72500 17.30200 + O -5.14800 -3.20300 19.40400 + H -5.86600 -2.96500 19.99000 + H -5.11100 -2.49000 18.76700 + O -16.66200 5.91100 19.90400 + H -17.57100 6.13600 19.70700 + H -16.46300 6.39500 20.70500 + O -3.91900 11.96900 17.16100 + H -3.63200 12.88200 17.17600 + H -3.10700 11.46200 17.15200 + O -11.16300 14.82700 20.64700 + H -11.42500 14.79700 19.72600 + H -11.39700 15.70900 20.93400 + O -8.76100 14.88400 17.46500 + H -8.22100 14.13600 17.21100 + H -8.50600 15.06800 18.36900 + O -8.43400 15.15300 20.21700 + H -9.34800 15.02400 20.47200 + H -7.94100 14.56000 20.78300 + O -7.63300 6.52700 15.85600 + H -7.79700 6.51200 14.91300 + H -6.75400 6.89600 15.94100 + O -12.84500 0.56700 17.23000 + H -13.09500 0.50700 18.15200 + H -13.31400 -0.15300 16.80700 + O -11.24100 3.82400 20.18700 + H -11.38400 4.61900 19.67400 + H -10.49000 3.40200 19.77100 + O -16.39900 0.64800 16.78700 + H -16.58300 0.42300 17.69900 + H -16.17400 1.57800 16.80900 + O -20.40900 3.37500 20.10800 + H -19.63800 3.77600 19.70700 + H -21.04700 3.31700 19.39800 + O -15.84100 2.38100 19.37000 + H -15.13500 2.73500 19.91200 + H -15.85900 2.94800 18.59900 + O -15.75700 3.39100 16.78100 + H -16.37000 3.61800 16.08200 + H -14.95100 3.85500 16.55400 + O -16.45900 0.38200 -21.60200 + H -17.28700 0.86200 -21.58600 + H -16.40700 0.02200 -22.48800 + O -13.47800 8.28400 19.29400 + H -14.20300 7.85200 18.84200 + H -13.89500 8.77900 19.99900 + O -10.22300 0.41100 18.10500 + H -11.12300 0.50700 17.79300 + H -9.92200 1.30800 18.25100 + O -9.01900 2.76300 18.89700 + H -8.19100 2.44100 19.25500 + H -8.80100 3.61300 18.51500 + O -11.77400 9.92800 17.90000 + H -12.18200 10.77900 18.05700 + H -12.36700 9.29800 18.30800 + O -5.55200 3.04900 17.77700 + H -5.97200 3.01500 16.91800 + H -5.20600 3.93900 17.84000 + O -4.83200 12.26700 19.79000 + H -5.65400 11.83100 20.01600 + H -4.74100 12.13400 18.84700 + O -20.02300 4.85600 17.16300 + H -19.67700 5.54500 17.73000 + H -20.87500 4.64000 17.54200 + O -16.00700 14.67400 -21.75600 + H -16.67000 14.28900 -21.18400 + H -15.30600 14.94500 -21.16400 + O -7.34400 10.02700 17.03200 + H -7.15200 9.46900 17.78600 + H -8.17400 9.69300 16.69200 + O -3.20100 14.59000 17.57400 + H -2.56300 15.29200 17.44100 + H -3.98200 15.03600 17.90000 + O -6.90300 8.50300 19.30100 + H -7.62200 7.93300 19.57100 + H -6.11100 8.01300 19.52200 + O -13.66500 10.99500 15.44200 + H -13.64000 10.13600 15.02000 + H -14.52600 11.03300 15.85800 + O -16.42900 10.83700 19.76800 + H -15.79900 10.48700 20.39800 + H -16.15500 11.74300 19.63200 + O -19.10700 6.90300 18.85700 + H -19.80300 7.35900 19.33000 + H -18.58800 7.60200 18.45900 + O -8.85600 -0.16600 20.42700 + H -9.32400 -0.08500 19.59600 + H -8.18000 0.51000 20.38800 + O -12.88200 12.23800 18.83800 + H -13.71500 12.51000 19.22400 + H -12.44700 13.05800 18.60400 + O -13.56500 2.61000 21.01900 + H -12.71600 2.99000 20.79400 + H -13.44400 2.25900 21.90200 + O -11.32800 -0.81400 21.57100 + H -11.34000 -0.65200 22.51500 + H -10.44600 -0.56200 21.29600 + O -15.26600 13.38600 19.79000 + H -15.08000 14.11300 19.19500 + H -15.52700 13.80800 20.60900 + O -16.19300 -0.25200 19.58200 + H -16.12300 0.70300 19.55900 + H -15.28700 -0.55900 19.56000 + O -17.80200 9.19400 17.87400 + H -17.38300 9.77500 18.50900 + H -17.14800 9.08200 17.18400 + O -20.36500 2.36200 15.65500 + H -20.25600 3.11900 16.23000 + H -21.16600 1.93900 15.96500 + O -19.20300 10.99100 20.33100 + H -19.49800 11.67000 19.72400 + H -18.26000 10.92800 20.18100 + O -11.89700 13.03500 15.71200 + H -12.53800 12.32400 15.69900 + H -11.06600 12.61400 15.49300 + O -7.31000 12.74200 16.49500 + H -7.15600 11.85700 16.82500 + H -6.66200 12.85600 15.80000 + O -15.90900 8.52200 15.96900 + H -15.66300 7.85900 16.61400 + H -15.17300 8.54800 15.35800 + O -9.82900 9.38300 16.03800 + H -10.58900 9.48900 16.61000 + H -10.08000 8.68800 15.43000 + O -15.11300 6.66300 17.79500 + H -15.65900 6.24000 18.45700 + H -14.66700 5.93900 17.35400 + O -13.54800 4.97300 16.20000 + H -13.62300 5.33700 15.31700 + H -12.61200 4.81500 16.31600 + O -5.04800 7.44500 15.83400 + H -4.98100 7.11700 14.93700 + H -4.91000 8.38900 15.75600 + O -14.26100 15.36600 18.12900 + H -13.31200 15.23500 18.14500 + H -14.42100 15.87000 17.33100 + O -20.42200 9.46400 16.93300 + H -19.52000 9.50300 17.25100 + H -20.37000 8.95500 16.12400 + O -4.65300 7.19400 20.19300 + H -3.93700 7.80100 20.00500 + H -4.22300 6.42200 20.56000 + O -3.51600 4.78600 21.04600 + H -4.14600 4.15500 21.39500 + H -2.94600 4.99100 21.78700 + O -4.53500 5.64400 17.81500 + H -4.74300 6.27300 17.12400 + H -4.65800 6.13300 18.62900 + O -6.59900 2.69900 15.22900 + H -6.20600 2.87500 14.37400 + H -6.59100 1.74500 15.30500 + O -6.66100 -0.03200 15.58200 + H -6.19100 -0.44500 16.30600 + H -7.57300 -0.29600 15.70500 + O -6.63100 1.57000 19.87300 + H -6.02100 0.91500 20.21300 + H -6.16600 1.97300 19.14000 + O -5.09800 -0.43900 21.19100 + H -4.72400 -0.40300 22.07200 + H -5.75300 -1.13600 21.23600 + O -4.87300 -1.24200 17.39500 + H -4.30100 -1.65200 16.74600 + H -4.41000 -0.44600 17.65400 + O -19.11900 1.43000 21.68600 + H -19.14900 0.63000 21.16200 + H -19.62300 2.06700 21.18000 + O -18.84600 -0.81600 20.09200 + H -17.93300 -0.73200 19.81700 + H -19.09200 -1.70600 19.84000 + O -7.20000 -2.29000 20.99100 + H -7.80000 -1.55800 20.85000 + H -7.76800 -3.05800 21.05300 + O -19.79500 12.96100 18.53200 + H -19.98700 13.12700 17.60900 + H -19.57800 13.82100 18.89100 + O -13.39500 0.07800 19.89200 + H -12.72500 -0.31600 20.45100 + H -13.53900 0.94800 20.26600 + O -11.53800 14.61200 17.88900 + H -10.62600 14.87300 17.76500 + H -11.76800 14.15300 17.08100 + O -4.34800 14.74000 20.90700 + H -4.46000 13.82700 20.64200 + H -4.60500 15.24700 20.13700 + O -1.09400 3.77000 19.90900 + H -1.97000 4.12400 20.06600 + H -1.17200 3.29700 19.08100 + O 8.81200 9.21600 20.80100 + H 8.41000 9.28300 19.93500 + H 8.07500 9.24600 21.41000 + O -2.71700 8.98600 19.39800 + H -2.62100 9.34800 18.51700 + H -2.51700 9.71800 19.98200 + O 10.77600 10.62700 -21.72400 + H 10.09500 10.19000 -22.23500 + H 10.76100 10.18500 -20.87500 + O 7.15500 5.65800 20.00500 + H 6.43900 6.26000 19.80300 + H 7.82900 5.86000 19.35600 + O 7.55900 6.13800 16.02000 + H 7.95600 6.14400 16.89100 + H 7.91500 5.35500 15.60000 + O 10.55100 7.13700 20.44700 + H 9.91300 7.83400 20.59700 + H 10.81800 6.86400 21.32500 + O 9.26900 6.04400 18.21900 + H 9.79700 6.37000 18.94800 + H 9.90200 5.87200 17.52200 + O 13.92000 -3.77400 17.05000 + H 13.44700 -3.23700 17.68500 + H 13.24800 -4.32900 16.65600 + O 1.02200 5.63700 19.69900 + H 0.44100 6.25800 19.25900 + H 0.47900 4.86100 19.84200 + O 15.45600 11.68900 16.32200 + H 15.46400 12.55100 16.73700 + H 16.17600 11.21500 16.73800 + O 7.14900 14.87900 19.76200 + H 7.06000 14.76200 18.81600 + H 6.96700 15.80800 19.90600 + O 10.10900 14.30500 16.88900 + H 10.56100 13.50400 16.62600 + H 10.25800 14.36500 17.83300 + O 9.90900 14.59600 19.71900 + H 8.95800 14.63200 19.81700 + H 10.21100 14.11500 20.49000 + O 10.87900 5.73900 15.89700 + H 10.87400 5.97500 14.96900 + H 11.73200 6.03500 16.21500 + O 7.30500 -0.02800 16.69000 + H 6.83100 0.05500 17.51700 + H 6.80700 -0.67600 16.19200 + O 8.61300 3.38900 20.85000 + H 8.19100 4.17400 20.50100 + H 9.25200 3.14200 20.18100 + O 3.89700 0.50300 16.20900 + H 3.81400 0.37200 17.15400 + H 3.19700 1.11700 15.99000 + O -3.59200 1.13400 17.96500 + H -2.78800 1.61200 17.76200 + H -4.28600 1.78800 17.88700 + O 3.58400 2.30200 20.61200 + H 4.53200 2.31600 20.74300 + H 3.44700 2.78300 19.79600 + O 3.39400 3.29300 17.95400 + H 2.77100 3.00000 17.28900 + H 4.06200 3.77200 17.46300 + O 1.65400 -2.47400 21.57900 + H 1.02100 -2.30300 20.88200 + H 2.32300 -1.79900 21.46700 + O 5.16200 7.54400 19.27000 + H 4.42700 7.25700 18.72700 + H 4.76300 8.10000 19.94000 + O 9.75300 -0.00000 17.93700 + H 8.89700 -0.05700 17.51300 + H 9.86800 0.93200 18.12100 + O 10.58200 2.48900 18.93000 + H 11.22900 2.17800 19.56200 + H 11.07300 2.59100 18.11400 + O 7.27900 9.19700 18.49300 + H 6.92100 10.08400 18.52500 + H 6.53300 8.63000 18.69200 + O 14.17700 2.80800 18.61600 + H 13.59800 2.63400 17.87500 + H 14.32700 3.75300 18.58700 + O 12.97200 11.15600 20.62300 + H 12.26100 10.95300 21.23200 + H 12.97900 10.42200 20.00900 + O -1.93700 5.38000 16.83900 + H -1.42500 6.02600 17.32600 + H -2.80600 5.41400 17.23800 + O 3.02600 14.21700 21.19900 + H 2.24700 14.72300 20.96800 + H 3.13200 14.35700 22.14000 + O 11.58500 9.63300 16.93800 + H 11.89100 9.32300 17.79000 + H 10.72900 9.22000 16.82600 + O 15.62200 13.99800 17.81200 + H 16.25000 14.71600 17.89200 + H 14.77100 14.40200 17.98300 + O 12.58600 8.64700 19.33400 + H 11.88600 8.10700 19.70100 + H 13.39300 8.25300 19.66700 + O 4.45000 10.55700 16.01100 + H 4.49200 9.71500 15.55800 + H 4.11100 10.34500 16.88000 + O 3.22200 10.33600 18.56400 + H 3.48200 9.96500 19.40700 + H 3.34700 11.27900 18.66800 + O -0.70800 7.34300 18.40700 + H -1.38700 7.80300 18.90000 + H -0.15100 8.03900 18.05800 + O 10.68900 -0.70500 20.38700 + H 10.32000 -0.50300 19.52700 + H 11.27900 0.02600 20.57200 + O 6.35600 11.74800 18.61700 + H 5.47300 12.06700 18.80500 + H 6.82900 12.52000 18.30700 + O 6.31700 1.79700 21.01200 + H 7.08000 2.37400 21.03000 + H 6.45700 1.19200 21.74000 + O 8.04600 -1.67300 20.72100 + H 7.54500 -1.32400 21.45800 + H 8.92600 -1.31500 20.83900 + O 3.88300 12.97700 18.91900 + H 4.04300 13.71500 18.33100 + H 3.50100 13.37200 19.70300 + O 3.41000 0.03600 18.86600 + H 3.31300 0.75800 19.48700 + H 4.29800 -0.29000 19.01400 + O 0.79400 9.49100 17.43300 + H 1.53000 9.84900 17.92900 + H 1.19800 9.05600 16.68300 + O -1.43400 2.70100 17.35600 + H -1.48700 3.58400 16.98800 + H -0.98900 2.18700 16.68200 + O -2.27200 11.50300 20.44900 + H -3.15600 11.84500 20.31300 + H -1.71900 12.03000 19.87300 + O 6.71000 12.05900 15.33200 + H 5.98200 11.51300 15.62600 + H 7.40500 11.43800 15.11400 + O 11.28700 12.08200 15.74000 + H 11.51000 11.29900 16.24300 + H 11.99300 12.16600 15.10000 + O 2.11400 7.91400 15.60400 + H 2.31800 7.38200 16.37300 + H 2.95600 8.03600 15.16500 + O 8.98300 8.71000 16.40700 + H 8.43900 8.92800 17.16400 + H 8.63400 7.87600 16.09300 + O 2.97900 6.50300 17.84000 + H 2.42400 6.03300 18.46300 + H 3.51200 5.82100 17.43200 + O 4.86300 5.08500 16.33300 + H 4.65600 5.14500 15.40000 + H 5.73100 5.48100 16.41200 + O 13.39700 6.85600 16.20800 + H 13.61000 6.59000 15.31300 + H 13.54700 7.80100 16.21900 + O 4.79400 15.01200 17.32200 + H 5.68400 14.68200 17.20200 + H 4.67900 15.65500 16.62200 + O -1.87400 10.11200 16.99900 + H -0.93500 10.09400 17.18400 + H -1.96200 9.66400 16.15800 + O 14.81400 7.42200 20.43500 + H 15.60200 7.95300 20.32200 + H 15.11200 6.64700 20.91100 + O 15.56500 5.03300 -22.29400 + H 15.00300 4.35900 -21.91300 + H 16.13300 5.30700 -21.57400 + O 14.53700 5.53200 18.34300 + H 14.10000 5.97700 17.61700 + H 14.50600 6.16200 19.06300 + O 12.32700 1.98500 16.73700 + H 12.30400 2.15100 15.79500 + H 12.60400 1.07200 16.81300 + O 13.58000 -1.22900 14.39100 + H 13.37600 -0.98100 15.29200 + H 12.91300 -0.78900 13.86400 + O 12.65400 1.34600 20.41400 + H 13.22500 0.86100 21.01100 + H 13.25500 1.80500 19.82700 + O 15.68700 -1.92500 20.70800 + H 16.31400 -1.61100 21.35900 + H 14.89900 -1.40400 20.86400 + O 13.40600 -0.58400 17.13700 + H 14.32600 -0.79900 17.28800 + H 12.92300 -1.16200 17.72700 + O -2.66200 -0.71400 19.80100 + H -3.43400 -0.76400 20.36500 + H -2.87200 -0.02800 19.16700 + O -0.25400 -2.01800 19.62600 + H 0.13300 -1.54500 18.88900 + H -1.09500 -1.58500 19.77400 + O 12.32200 -2.45400 18.94100 + H 11.79300 -1.88200 19.49800 + H 12.09300 -3.34000 19.22000 + O -1.01300 13.27300 18.66600 + H -1.76200 13.78100 18.35400 + H -0.44500 13.91800 19.08700 + O 1.10500 -0.73200 17.57400 + H 1.44800 -1.38300 16.96200 + H 1.87100 -0.44300 18.07100 + O 6.14600 -0.24400 19.17100 + H 6.69600 -0.92600 19.55900 + H 6.23100 0.50000 19.76600 + O 7.35900 14.02900 17.10300 + H 8.28300 14.23300 16.95900 + H 7.15000 13.37800 16.43200 + O 13.56200 13.82500 20.84800 + H 13.53000 12.87600 20.72700 + H 13.51600 14.18200 19.96100 + O -0.26400 12.87100 15.93700 + H -0.47100 12.83500 16.87100 + H 0.68100 12.72100 15.89600 + O 17.48400 3.51900 20.31800 + H 16.77500 4.01900 20.72200 + H 17.08000 2.68800 20.06800 + O 16.95400 9.08100 19.94200 + H 16.93600 9.45500 19.06100 + H 17.27600 9.79200 20.49700 + O 19.80900 4.80500 19.78800 + H 19.46200 5.65800 19.52600 + H 19.03400 4.28800 20.00800 + O 16.46700 1.26900 18.79400 + H 17.11500 1.70200 18.23900 + H 15.67100 1.78900 18.68400 + O 21.60500 3.66600 18.07800 + H 21.14200 3.44500 17.27000 + H 20.93300 4.04400 18.64500 + O 17.14900 5.22200 17.45400 + H 17.68400 5.84800 17.94200 + H 16.29000 5.26000 17.87600 + O 18.61700 7.11200 18.86000 + H 18.10900 7.69400 19.42500 + H 19.15300 7.70000 18.32800 + O 19.86600 8.90000 17.13600 + H 20.39400 9.69500 17.06700 + H 20.12300 8.37500 16.37700 + O 18.22800 2.69100 17.31000 + H 17.86800 3.57300 17.40200 + H 18.96800 2.79500 16.71200 + O 17.82500 11.33400 21.30300 + H 18.43400 11.64400 21.97300 + H 17.88400 11.98600 20.60500 + O 20.73600 7.71600 14.80400 + H 20.70900 6.80400 14.51300 + H 21.66400 7.94700 14.78300 + O 17.31700 9.88600 17.32000 + H 18.25400 9.69000 17.33700 + H 16.97200 9.34800 16.60800 + O 16.00000 -1.30700 17.99900 + H 15.87700 -1.71200 18.85700 + H 16.26200 -0.40700 18.19300 + O 17.87300 13.17200 19.25400 + H 17.02600 13.04600 18.82600 + H 18.01600 14.11800 19.23100 + O 19.69300 13.17000 17.11100 + H 19.23800 13.01500 17.93900 + H 20.23800 12.39300 16.98800 + O -5.22900 16.16900 18.65300 + H -5.90800 16.56700 19.19900 + H -4.98300 16.85900 18.03700 + O -12.39700 18.45900 17.19700 + H -12.83600 18.61900 18.03200 + H -12.91400 17.76800 16.78200 + O -15.55600 19.17800 15.26000 + H -16.33800 18.69200 15.52100 + H -15.85100 20.08500 15.18200 + O -19.93100 19.37700 16.79900 + H -19.14500 18.92100 16.49800 + H -19.62200 20.24800 17.04900 + O -14.99900 21.14800 20.93100 + H -14.05700 21.28600 21.03700 + H -15.06700 20.43400 20.29700 + O -18.27400 16.30000 21.60000 + H -18.51000 16.04700 20.70700 + H -17.48000 15.80000 21.78900 + O -9.86200 19.32300 17.85000 + H -10.75200 19.02600 17.66000 + H -9.74200 19.14200 18.78200 + O -8.22800 21.58800 19.99700 + H -7.33200 21.74300 20.29400 + H -8.23100 21.87000 19.08300 + O -9.25900 19.22300 20.78500 + H -8.87800 20.02300 20.42500 + H -9.59900 19.48300 21.64100 + O -12.22500 21.20400 21.11300 + H -11.92400 22.10500 20.99500 + H -11.62200 20.83300 21.75800 + O -11.50100 17.46000 21.36800 + H -11.76300 17.71100 22.25400 + H -10.69400 17.94900 21.20900 + O -20.18800 22.15300 14.77900 + H -20.03400 22.14000 15.72300 + H -19.71300 21.39200 14.44700 + O -6.39000 19.95500 16.46100 + H -5.73900 19.26600 16.59600 + H -7.04000 19.56100 15.88000 + O -5.63700 21.86700 20.89600 + H -5.29600 21.22500 21.51900 + H -5.19800 21.65700 20.07100 + O -4.02700 17.53600 21.21900 + H -3.95200 16.61400 21.46500 + H -4.93300 17.63400 20.92600 + O -4.37300 18.14400 16.97300 + H -3.96700 17.68800 16.23500 + H -3.64400 18.57500 17.41900 + O -20.93200 17.70300 18.85700 + H -21.81700 17.96900 18.60600 + H -20.35600 18.24200 18.31400 + O -19.14000 15.61000 19.05100 + H -18.71600 15.65200 18.19400 + H -19.85700 16.24100 18.99600 + O -6.91700 17.45000 20.42700 + H -7.52100 18.17100 20.60000 + H -7.46800 16.66700 20.43400 + O -13.18900 19.01500 19.75200 + H -12.83500 18.30200 20.28300 + H -12.81200 19.80800 20.13400 + O 13.33900 15.43300 18.46200 + H 12.59300 15.85600 18.88700 + H 13.74300 16.12600 17.94000 + O 6.89000 18.32500 16.16000 + H 6.35000 18.62400 16.89200 + H 6.41300 17.57700 15.80000 + O 3.75600 19.06300 14.72400 + H 3.53300 19.17300 15.64800 + H 3.05100 19.50700 14.25200 + O -2.53000 19.69200 18.35900 + H -1.93100 20.22400 17.83600 + H -3.27900 20.26600 18.52400 + O 3.84800 -22.02000 17.13200 + H 4.29300 -22.05500 17.97800 + H 4.31400 -21.34000 16.64400 + O 1.26700 19.46900 19.82900 + H 0.47300 18.94900 19.94700 + H 1.57100 19.24700 18.94900 + O 9.36200 18.89600 17.17200 + H 8.49000 18.67100 16.84900 + H 9.33300 18.68700 18.10600 + O 9.76600 21.43900 19.39100 + H 10.46300 21.80200 19.93700 + H 10.01900 21.66700 18.49600 + O 9.47300 18.84900 20.11900 + H 9.53200 19.75300 19.80900 + H 9.40600 18.92600 21.07000 + O 5.06800 21.42100 19.59600 + H 5.60000 22.08800 20.02900 + H 4.51600 21.06500 20.29200 + O 6.98300 17.61100 20.24000 + H 6.68900 17.96700 21.07800 + H 7.87900 17.93400 20.14300 + O 2.90300 19.25700 17.38400 + H 2.99600 20.20800 17.32900 + H 3.71600 18.95900 17.79300 + O -1.01000 21.42300 16.94800 + H -0.69000 22.14900 17.48400 + H -1.26400 21.83100 16.12000 + O 12.84300 19.58600 16.67500 + H 13.46200 18.88400 16.87800 + H 12.31800 19.24100 15.95300 + O 12.07300 21.86300 20.88300 + H 12.08600 21.10800 21.47100 + H 12.67100 21.62600 20.17500 + O 14.07400 16.64300 21.14400 + H 14.07900 15.68600 21.15500 + H 13.23800 16.87400 20.73900 + O 14.62100 17.49000 17.15900 + H 15.15200 17.07300 16.48100 + H 15.23900 18.03300 17.64900 + O -1.29900 17.95300 20.06200 + H -1.99300 17.72100 20.67800 + H -1.69700 18.60700 19.48700 + O -1.17400 16.39800 17.69300 + H -0.43100 16.72300 17.18400 + H -1.16100 16.92100 18.49500 + O 11.48900 16.84600 19.96500 + H 10.91100 17.60900 19.98700 + H 10.89400 16.09600 19.94900 + O 1.03800 17.36600 16.29300 + H 1.50700 16.65200 15.86000 + H 1.72800 17.91600 16.66500 + O 5.43500 18.96000 18.39700 + H 5.87800 18.40000 19.03500 + H 5.44100 19.83000 18.79500 + O 15.88000 19.48700 18.67700 + H 16.46000 19.94400 18.06900 + H 15.17500 20.11000 18.85100 + O -21.10300 16.90000 21.51200 + H -20.21300 16.62200 21.72600 + H -21.05000 17.19800 20.60400 + O 21.21500 19.15400 17.17300 + H 20.78100 19.63200 16.46600 + H 22.14100 19.38000 17.08200 + O 17.13800 21.10400 16.91700 + H 17.39600 21.96400 17.24800 + H 16.78000 21.27900 16.04700 + O 16.64600 17.57900 20.48400 + H 15.79700 17.21900 20.74100 + H 16.43100 18.30500 19.89800 + O 17.61100 15.84800 18.49100 + H 18.28000 16.25800 17.94200 + H 17.44600 16.48600 19.18500 + O 19.60700 16.93600 16.83500 + H 20.24000 16.28200 16.53600 + H 20.14100 17.69100 17.08300 diff --git a/examples/gpmdk/run/TrpCage/sbatch.sh b/examples/gpmdk/run/TrpCage/sbatch.sh new file mode 100755 index 00000000..0d0710b3 --- /dev/null +++ b/examples/gpmdk/run/TrpCage/sbatch.sh @@ -0,0 +1,19 @@ +#!/bin/bash +##SBATCH -p gpu +##SBATCH -p gpu_debug +##SBATCH --reservation=gpu_debug +##SBATCH --qos=debug +#SBATCH --time 2:00:00 +#SBATCH -N 64 +##SBATCH -A ichelp_g +##SBATCH -A w23_macroqmd_g + +cd ${PWD} +source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh + +#export MPICH_ALLREDUCE_NO_SMP=1 +export MPICH_SMP_SINGLE_COPY_MODE=NONE +#export MPICH_OPT_THREAD_SYNC=0 + +OMP_NUM_THREADS=72 srun -n 256 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh + diff --git a/examples/gpmdk/run/TrpCage/sbatch_profile.sh b/examples/gpmdk/run/TrpCage/sbatch_profile.sh new file mode 100755 index 00000000..3ebf58e8 --- /dev/null +++ b/examples/gpmdk/run/TrpCage/sbatch_profile.sh @@ -0,0 +1,18 @@ +#!/bin/bash +##SBATCH -p gpu +##SBATCH -p gpu_debug +##SBATCH --reservation=gpu_debug +##SBATCH --qos=debug +##SBATCH --time 2:00:00 +#SBATCH -N 32 +##SBATCH -A ichelp_g +##SBATCH -A w23_macroqmd_g + +cd ${PWD} +source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh + +#export MPICH_ALLREDUCE_NO_SMP=1 +export MPICH_SMP_SINGLE_COPY_MODE=NONE + +OMP_NUM_THREADS=72 srun -n 128 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper_profile.sh + diff --git a/examples/gpmdk/run/TrpCage/wrapper.sh b/examples/gpmdk/run/TrpCage/wrapper.sh new file mode 100644 index 00000000..66b07acf --- /dev/null +++ b/examples/gpmdk/run/TrpCage/wrapper.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) + +#RUN="nsys profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_gpu/gpmdcov" +#RUN="map --profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_forge/gpmdcov" +RUN="/usr/projects/icapt/mewall/venado/packages/qmd-progress/examples/gpmdk/build_hackathon/gpmdk" +#stdbuf -o0 $RUN input_ICH.in |& tee out_ICH +if [ $SLURM_PROCID -eq 0 ]; then + stdbuf -o0 $RUN input.in >& out_0 +else + $RUN input.in >& /dev/null +fi diff --git a/examples/gpmdk/run/TrpCage/wrapper_profile.sh b/examples/gpmdk/run/TrpCage/wrapper_profile.sh new file mode 100644 index 00000000..6c4fb53c --- /dev/null +++ b/examples/gpmdk/run/TrpCage/wrapper_profile.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) + +#RUN="nsys profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_gpu/gpmdcov" +#RUN="map --profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_forge/gpmdcov" +RUN="/usr/projects/icapt/mewall/venado/packages/qmd-progress/examples/gpmdk/build_hackathon/gpmdk" +#stdbuf -o0 $RUN input_ICH.in |& tee out_ICH +if [ $SLURM_PROCID -eq 0 ]; then + nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -t mpi,openmp,cuda,nvtx --mpi-impl=mpich $RUN input.in >& out_0 +else + $RUN input.in >& /dev/null +fi From 639473fb9aba6e8db7031e867c1a38fef143c902 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Mon, 8 Jul 2024 10:45:57 -0600 Subject: [PATCH 10/69] Fixed line truncations --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 004d9866..0ba912da 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -105,21 +105,25 @@ subroutine gpmdcov_EnergAndForces(charges) if(gpmdt%usevectsk)then - call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& - syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & - lt%threshold, dH0x_bml,dH0y_bml,dH0z_bml) + call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& + &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) - call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& - syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & - lt%threshold, dSx_bml,dSy_bml,dSz_bml) + call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + &syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& + &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + <%threshold, dSx_bml,dSy_bml,dSz_bml) else - call get_dH(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& - syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & - lt%threshold, dH0x_bml,dH0y_bml,dH0z_bml) - - call get_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& - syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & - lt%threshold, dSx_bml,dSy_bml,dSz_bml) + call get_dH(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& + &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) + + call get_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + &syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& + &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + <%threshold, dSx_bml,dSy_bml,dSz_bml) endif if(printRank() == 1 .and. lt%verbose >= 10)then @@ -206,7 +210,9 @@ subroutine gpmdcov_EnergAndForces(charges) call gpmdcov_msI("gpmdcov_EnergAndForces","SMD Total Energy " & & // to_string(smd_total_energy),lt%verbose,myRank) do k = 1,3 - dcoords(k) = modulo((R1(k) - R2(k)) + 0.5_dp*sy%lattice_vector(k,k),sy%lattice_vector(k,k)) - 0.5_dp * sy%lattice_vector(k,k) + dcoords(k) = modulo((R1(k) - R2(k)) + & + &0.5_dp*sy%lattice_vector(k,k),sy%lattice_vector(k,k)) - & + &0.5_dp * sy%lattice_vector(k,k) enddo dist = norm2(dcoords) call gpmdcov_msI("gpmdcov_EnergAndForces","SMD distance " & From 61a5be87f76204e1f4179dd45b7b9b52ebf2a8f0 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Tue, 16 Jul 2024 22:30:24 -0600 Subject: [PATCH 11/69] Added sedacs partition and field induced forces --- .../gpmdk/run/TrpCage/input_sedacsPart.in | 201 +++++++++ examples/gpmdk/src/gpmd.py | 51 ++- examples/gpmdk/src/gpmd_secuential.F90 | 9 +- examples/gpmdk/src/gpmdcov_graphpart.F90 | 18 +- examples/gpmdk/src/gpmdcov_highorder.F90 | 47 ++ examples/gpmdk/src/gpmdcov_init.F90 | 10 + examples/gpmdk/src/gpmdcov_lib.F90 | 43 +- examples/gpmdk/src/gpmdcov_lib_mod.F90 | 15 +- examples/gpmdk/src/gpmdcov_neighbor.F90 | 136 +++++- examples/gpmdk/src/gpmdcov_vars.F90 | 1 + src/prg_graph_mod.F90 | 422 +++++++++++++++++- src/prg_graphsp2parser_mod.F90 | 16 +- src/prg_system_mod.F90 | 1 + 13 files changed, 910 insertions(+), 60 deletions(-) create mode 100644 examples/gpmdk/run/TrpCage/input_sedacsPart.in diff --git a/examples/gpmdk/run/TrpCage/input_sedacsPart.in b/examples/gpmdk/run/TrpCage/input_sedacsPart.in new file mode 100644 index 00000000..58031cda --- /dev/null +++ b/examples/gpmdk/run/TrpCage/input_sedacsPart.in @@ -0,0 +1,201 @@ +INPUT FILE FOR THE GPMD PROGRAM +=============================== + +#LATTE parameters +Latte{ + JobName= GPMD + #BMLType= Ellpack + BMLType= Dense + #Method= GSP2 + #Method= SP2 + #Method= Diag + #Method= DiagEf + Method= DiagEfFull + MDim= -1 + #Threshold= 1.0d-5 + Threshold= 0.0 + Verbose= 5 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + #SCF variables# + #StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_DM_Min" + #StopAt= "gpmdcov_FirstCharges" + MPulay= 10 + #ZMat= ZSP + ZMat= Diag + PulayCoeff= 0.1 + #MixCoeff= 0.6 #VALID FOR WAT + MixCoeff= 0.2 + SCFTol= 1.0d-8 + MaxSCFIter= 500 + CoulAcc= 1.0d-5 + TimeRatio= 10.0 + TimeStep= 0.25 + #TimeStep= 0.00 + MDSteps= 1000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + #CoordsFile= coords_300.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb + #CoordsFile= minimized_TrpCage.dat + #CoordsFile= Mac1_latte.dat + #CoordsFile= ICH.dat + CoordsFile= input1.dat + NlistEach= 1 + #BMLDistributionType= Distributed + MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn + EFermi= -0.0 + #kBT= 0.04308695 + kBT= 0.025 + Entropy= T + DoKernel= T +} + +#SP2 Solver +SP2{ + MinSP2Iter= 10 + MaxSP2Iter= 200 + SP2Tol= 1.0d-5 + SP2Conv= Rel +} + +#Graph-based SP2 parameters +GSP2{ + BMLType= Ellpack + #GraphElement= Orbital + GraphElement= Atom + #PartitionType= Block + #NodesPerPart= 333 + #NodesPerPart= 18 + #NodesPerPart= 27 + #NodesPerPart= 512 + #NodesPerPart= 17 + #NodesPerPart= 48 + #NodesPerPart= 150 + #NodesPerPart= 1331 + PartitionType= Sedacs + #PartitionType= METIS + #PartitionType= METIS+SA + #PartitionType= METIS+KL + #PartitionRefinement= None + #PartitionRefinement= KL + #PartitionRefinement= SA + NLGraphCut= 4.5 + CovGraphFact= 4.5 + #CovGraphFact= 3.0 + #PartitionCount= 64 + PartitionCount= 256 + PartitionCountX= 8 + PartitionCountY= 8 + PartitionCountZ= 4 + #PartitionCount= 3200 + #PartitionCount= 6400 + #PartitionCount= 1 + GraphThreshold= 0.0005 + ErrLimit= 1.0e-12 + PartEach= 100 + Mdim= 4000 +} + + +#Sparse propagation of the inverse overlap +ZSP{ + Verbose= 1 + NFirst= 8 + NRefI= 3 + NRefF= 1 + Int= .true. + NumthreshI= 1.0d-8 + NumthreshF= 1.0d-5 +} + +#Extended Lagrangian parameters +XLBO{ + JobName= XLBO + Verbose= 1 + Mprg_init= 2 + MaxSCFIter= 0 + MaxSCFInitIter= 50 + NumThresh= 0.0 +} + + +KERNEL{ + KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks + BuildAlways= F + RankNUpdate= 3 + KernelMixing= T + InitialMixingWith= DIIS + UpdateEach= 1 + UpdateAfterBuild= T + Verbose= 2 +} + +GPMD{ + DoVelocityRescale= F + #VRFactor= 1.0 + WriteTrajectory= T + TrajectoryFormat= XYZ + WriteCoordsEach= 1 + InitialTemperature= 300 + DumpEach= 100 + RestartFromDump= F + #MinimizationSteps= 500 + LangevinMethod= Siva + LangevinDynamics= F + LangevinGamma= 0.01 + #SMDForceConstantStart= 0.1 + #SMDForceConstantEnd= 0.1 + #SMDR0= 2.0 + #SMDNumPairs= 1 + #SMDPairs[ + #127 317 + #] + #SMDAtomIndex1= 324 + #SMDAtomIndex2= 315 + UseSMD= F + HtoD= F + UseVectSKBlock= F + ReplicateX= 1 + ReplicateY= 1 + ReplicateZ= 1 +} diff --git a/examples/gpmdk/src/gpmd.py b/examples/gpmdk/src/gpmd.py index e7c05e1f..85171b35 100755 --- a/examples/gpmdk/src/gpmd.py +++ b/examples/gpmdk/src/gpmd.py @@ -16,18 +16,13 @@ # @param coords for each atom, e.g., z-coordinate of the frist atom is coords[0,2] # @param verb Verbosity level for gpmd output # -def gpmd(latticeVectors,symbols,atomTypes,coords,verb): +def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): # Import the shared library gpmdLibFileName = os.environ['GPMD_PATH'] + '/libgpmd.so' - try: - gpmdLib = ct.CDLL(gpmdLibFileName) - f = gpmdLib.gpmd_compute - except: - print("ERROR loading the gpmd library. Please the check that the \ - GPMD_PATH is set correctly and that there is a compiled libgpmd.so") - exit(0) + gpmdLib = ct.CDLL(gpmdLibFileName) + f = gpmdLib.gpmd_compute #Periodic table: We use this to pass the chemical atom types as integer instead of characters. pt = ptable() @@ -48,7 +43,8 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,verb): forcesFlat_out = np.zeros(3*nats) #Vectorized forces chargesFlat_out = np.zeros(nats) #We call this one Flat just for consistency dipoleFlat_out = np.zeros(3) #Same here - bornchFlat_out = np.zeros(9*nats) + fieldFlat_in = np.zeros(3) #Same here + #bornchFlat_out = np.zeros(9*nats) for i in range(nats): coordsFlat_in[3*i] = coords[i,0] @@ -68,6 +64,8 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,verb): latticeVectorsFlat[7] = latticeVectors[2,1] latticeVectorsFlat[8] = latticeVectors[2,2] + fieldFlat_in[:] = field[:] + #Specify arguments as a pointers to pass to Fortran f.argtypes=[ct.c_int,ct.c_int,ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),\ ct.POINTER(ct.c_int),ct.POINTER(ct.c_int),ct.POINTER(ct.c_double),\ @@ -78,45 +76,46 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,verb): atomTypes_ptr = atomTypes32.ctypes.data_as(ct.POINTER(ct.c_int)) atomicNumbers_ptr = atomicNumbers.ctypes.data_as(ct.POINTER(ct.c_int)) latticeVectors_ptr = latticeVectorsFlat.ctypes.data_as(ct.POINTER(ct.c_double)) + field_ptr = fieldFlat_in.ctypes.data_as(ct.POINTER(ct.c_double)) #Outputs charges_ptr = chargesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) forces_ptr = forcesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) dipole_ptr = dipoleFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) - bornch_ptr = bornchFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + #bornch_ptr = bornchFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) #Call to the fortran funtion err = f(ct.c_int(nats),ct.c_int(nTypes),coords_ptr,latticeVectors_ptr,\ - atomTypes_ptr,atomicNumbers_ptr,charges_ptr,forces_ptr,\ - dipole_ptr,bornch_ptr,ct.c_int(verb)) + atomTypes_ptr,atomicNumbers_ptr,field_ptr,charges_ptr,forces_ptr,\ + dipole_ptr,ct.c_int(verb)) charges_out = np.zeros(nats) charges_out[:] = chargesFlat_out[:] dipole_out = np.zeros(3) dipole_out[:] = dipoleFlat_out[:] - + #Back to a 2D array for the forces forces_out = np.zeros((nats,3)) - bornch_out = np.zeros((nats,9)) + #bornch_out = np.zeros((nats,9)) for i in range(nats): forces_out[i,0] = forcesFlat_out[i*3 + 0] forces_out[i,1] = forcesFlat_out[i*3 + 1] forces_out[i,2] = forcesFlat_out[i*3 + 2] - bornch_out[i,0] = bornchFlat_out[i*9 + 0] - bornch_out[i,1] = bornchFlat_out[i*9 + 1] - bornch_out[i,2] = bornchFlat_out[i*9 + 2] - - bornch_out[i,3] = bornchFlat_out[i*9 + 3] - bornch_out[i,4] = bornchFlat_out[i*9 + 4] - bornch_out[i,5] = bornchFlat_out[i*9 + 5] - - bornch_out[i,6] = bornchFlat_out[i*9 + 6] - bornch_out[i,7] = bornchFlat_out[i*9 + 7] - bornch_out[i,8] = bornchFlat_out[i*9 + 8] + # bornch_out[i,0] = bornchFlat_out[i*9 + 0] + # bornch_out[i,1] = bornchFlat_out[i*9 + 1] + # bornch_out[i,2] = bornchFlat_out[i*9 + 2] + # + # bornch_out[i,3] = bornchFlat_out[i*9 + 3] + # bornch_out[i,4] = bornchFlat_out[i*9 + 4] + # bornch_out[i,5] = bornchFlat_out[i*9 + 5] + # + # bornch_out[i,6] = bornchFlat_out[i*9 + 6] + # bornch_out[i,7] = bornchFlat_out[i*9 + 7] + # bornch_out[i,8] = bornchFlat_out[i*9 + 8] - return err, charges_out, forces_out, dipole_out, bornch_out + return err, charges_out, forces_out, dipole_out diff --git a/examples/gpmdk/src/gpmd_secuential.F90 b/examples/gpmdk/src/gpmd_secuential.F90 index 9cfac1a2..38571ddd 100644 --- a/examples/gpmdk/src/gpmd_secuential.F90 +++ b/examples/gpmdk/src/gpmd_secuential.F90 @@ -26,7 +26,8 @@ program gpmd real(dp), allocatable :: lattice_vectors_in(:,:) real(dp), allocatable :: forces_out(:,:) real(dp), allocatable :: charges_out(:) - real(dp), allocatable :: borncharges_out(:,:) + !real(dp), allocatable :: borncharges_out(:,:) + real(dp), allocatable :: field_in(:) real(dp), allocatable :: dipole_out(:) type(system_type) :: mysys integer :: verb_in @@ -43,8 +44,10 @@ program gpmd atomic_numbers_in = mysys%spatnum lattice_vectors_in = mysys%lattice_vector verb_in = 3 + allocate(field_in(3)) + field_in = 0.0_dp call gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& - &charges_out,forces_out,dipole_out,borncharges_out,verb_in) + &field_in,charges_out,forces_out,dipole_out,verb_in) call prg_parse_system(mysys,"benzene-CN.pdb") deallocate(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in) @@ -62,7 +65,7 @@ program gpmd lattice_vectors_in = mysys%lattice_vector verb_in = 1 call gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& - &charges_out,forces_out,dipole_out,borncharges_out,verb_in) + &field_in,charges_out,forces_out,dipole_out,verb_in) end program gpmd diff --git a/examples/gpmdk/src/gpmdcov_graphpart.F90 b/examples/gpmdk/src/gpmdcov_graphpart.F90 index 66e54269..c855184b 100644 --- a/examples/gpmdk/src/gpmdcov_graphpart.F90 +++ b/examples/gpmdk/src/gpmdcov_graphpart.F90 @@ -22,7 +22,23 @@ subroutine gpmd_graphpart call prg_equalGroupPartition(gpat, hindex, nnodes, gsp2%nodesPerPart, sy%nats) endif - !> METIS, METIS+SA, or METIS+KL partitioning + + !> SEDACS partitioning + elseif (gsp2%partition_type == "Sedacs") then + if (gsp2%graph_element .ne. "Atom") then + call gpmdcov_message("graph_part","!!!ERROR, GraphElement needs to be set to Atom",lt%verbose,2,myrank) + stop + else + if(gsp2%nx*gsp2%ny*gsp2%nz .ne. 0)then + call gpmdcov_get_nlist_box_indices(sy%coordinate,whichParts_guess_saved,sy%lattice_vector,& + &gsp2%nx,gsp2%ny,gsp2%nz,lt%verbose) + gpat%TotalParts = maxval(whichParts_guess_saved) + endif + gpat%TotalParts = gsp2%partition_count + call prg_sedacsPartition(gpat,sy%coordinate,whichParts_guess_saved,g_bml,gpat%TotalParts,sy%nats,lt%verbose) + endif + + !> METIS, METIS+SA, or METIS+KL partitioning else #ifdef DO_GRAPHLIB diff --git a/examples/gpmdk/src/gpmdcov_highorder.F90 b/examples/gpmdk/src/gpmdcov_highorder.F90 index 535cc889..696b96ed 100644 --- a/examples/gpmdk/src/gpmdcov_highorder.F90 +++ b/examples/gpmdk/src/gpmdcov_highorder.F90 @@ -21,6 +21,7 @@ module gpmdcov_highorder_mod public :: gpmdcov_get_born_charges_v1 public :: gpmdcov_get_born_charges_v2 + public :: gpmdcov_field_forces contains @@ -245,4 +246,50 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) end subroutine gpmdcov_get_born_charges_v2 + + !> Field forces + subroutine gpmdcov_get_field_forces(norbi,field_in) + implicit none + real(dp),allocatable :: dipoleMoment(:),savedCharges(:) + real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) + real(dp) :: lambda,intensity,field(3),threshold + real(dp), allocatable :: forcesSaved(:,:) + real(dp), allocatable, intent(in) :: field_in(:) + integer :: atomI, verbose + integer, allocatable, intent(in) :: norbi(:) + type(bml_matrix_t) :: prt_bml + real(dp) :: relperm, keconst, tfact + real(dp) :: intensity0,dintensity + + lambda = 1.0_dp !2.0_dp/14.3996437701414_dp + threshold = 0.0_dp + verbose = 3 + + !This will assume only one part + call gpmdcov_InitParts() + allocate(forcesSaved(3,sy%nats)) + + intensity0 = 0.0000_dp !eV/Ang + dintensity = 0.001_dp + intensity = 1.0D-4 + + field = field_in/norm2(field_in) + intensity = norm2(field_in) + + call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) + call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + sy%net_charge = 0.0_dp + call gpmdcov_InitParts() + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V + call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call gpmdcov_Diagonalize_H0() + call gpmdcov_muFromParts() + call gpmdcov_FirstCharges(eig) + call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_EnergAndForces(sy%net_charge) + + end subroutine gpmdcov_get_field_forces + + end module gpmdcov_highorder_mod diff --git a/examples/gpmdk/src/gpmdcov_init.F90 b/examples/gpmdk/src/gpmdcov_init.F90 index db0b4114..b52986cb 100644 --- a/examples/gpmdk/src/gpmdcov_init.F90 +++ b/examples/gpmdk/src/gpmdcov_init.F90 @@ -59,6 +59,16 @@ subroutine gpmdcov_Init(lib_on) ! related variables incluind partition methos, number of parts, etc. call prg_parse_gsp2(gsp2,inputfile) + if((gsp2%nx .ne. 0) .and. (gsp2%ny .ne. 0) .and. (gsp2%nz .ne. 0))then + nparts = gsp2%nx*gsp2%ny*gsp2%nz + if(nparts .ne. gsp2%partition_count)then + call gpmdcov_msI("gpmdcov_Init","!!!ERROR: If PartitionCountX/Y/Z are set, PartitionCount & + &should be set to PartitionCountx * PartitionCountY * PartitionCountZ",lt%verbose,Myrank) + stop + endif + endif + + !> Parsing Extended Lagrangian input paramenters. This will read all the ! variables related to the Extended lagrangian method. ! xl is the "xlbo_type". diff --git a/examples/gpmdk/src/gpmdcov_lib.F90 b/examples/gpmdk/src/gpmdcov_lib.F90 index ac2155b6..d0245f11 100644 --- a/examples/gpmdk/src/gpmdcov_lib.F90 +++ b/examples/gpmdk/src/gpmdcov_lib.F90 @@ -26,7 +26,7 @@ !! \param verb_in Verbosity level. !! function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomicNumbers_in,& - &charges_out,forces_out,dipole_out,bornch_out,verb_in) result(err) bind(c, name='gpmd_compute') + &field_in,charges_out,forces_out,dipole_out,verb_in) result(err) bind(c, name='gpmd_compute') use iso_c_binding, only: c_char, c_double, c_int, c_bool use gpmdcov_vars @@ -36,10 +36,11 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi integer(c_int), intent(in), value :: nats integer(c_int), intent(in), value :: nTypes real(c_double), intent(inout) :: coords_in(3*nats) + real(c_double), intent(inout) :: field_in(3) real(c_double), intent(inout) :: forces_out(3*nats) real(c_double), intent(inout) :: charges_out(nats) real(c_double), intent(inout) :: dipole_out(3) - real(c_double), intent(inout) :: bornch_out(9*nats) + !real(c_double), intent(inout) :: bornch_out(9*nats) integer(c_int), intent(inout) :: atomTypes_in(nats) integer(c_int) ,intent(inout) :: atomicNumbers_in(nTypes) real(c_double) ,intent(inout) :: latticeVectors_in(9) @@ -48,7 +49,8 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi real(dp), allocatable :: coords(:,:) real(dp), allocatable :: forces(:,:), charges(:), dipole(:) - real(dp), allocatable :: bornch(:,:) + !real(dp), allocatable :: bornch(:,:) + real(dp), allocatable :: field(:) real(dp), allocatable :: latticeVectors(:,:) integer, allocatable :: atomTypes(:), atomicNumbers(:) integer :: k @@ -62,7 +64,8 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi allocate(charges(nats)) allocate(forces(3,nats)) allocate(dipole(3)) - allocate(bornch(9,nats)) + !allocate(bornch(9,nats)) + allocate(field(3)) !Note that arrays appear in another order. We need to rearange !the data. This is because of the column mayor (in python) vs. @@ -91,9 +94,11 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi atomTypes(k) = atomTypes_in(k) + 1 enddo + field = field_in + verb = verb_in call gpmd_compute(coords,atomTypes,atomicNumbers,latticeVectors,& - &charges,forces,dipole,bornch,verb) + &field,charges,forces,dipole,verb) err = .false. @@ -104,19 +109,19 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi forces_out((k-1)*3 + 3) = forces(3,k) enddo - do k = 1, nats - bornch_out((k-1)*9 + 1) = bornch(1,k) - bornch_out((k-1)*9 + 2) = bornch(2,k) - bornch_out((k-1)*9 + 3) = bornch(3,k) - - bornch_out((k-1)*9 + 4) = bornch(4,k) - bornch_out((k-1)*9 + 5) = bornch(5,k) - bornch_out((k-1)*9 + 6) = bornch(6,k) - - bornch_out((k-1)*9 + 7) = bornch(7,k) - bornch_out((k-1)*9 + 8) = bornch(8,k) - bornch_out((k-1)*9 + 9) = bornch(9,k) - enddo +! do k = 1, nats +! bornch_out((k-1)*9 + 1) = bornch(1,k) +! bornch_out((k-1)*9 + 2) = bornch(2,k) +! bornch_out((k-1)*9 + 3) = bornch(3,k) +! +! bornch_out((k-1)*9 + 4) = bornch(4,k) +! bornch_out((k-1)*9 + 5) = bornch(5,k) +! bornch_out((k-1)*9 + 6) = bornch(6,k) +! +! bornch_out((k-1)*9 + 7) = bornch(7,k) +! bornch_out((k-1)*9 + 8) = bornch(8,k) +! bornch_out((k-1)*9 + 9) = bornch(9,k) +! enddo charges_out(:) = charges(:) dipole_out(:) = dipole(:) @@ -128,7 +133,7 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi deallocate(atomTypes) deallocate(atomicNumbers) deallocate(dipole) - deallocate(bornch) + !deallocate(bornch) return diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 index 00ef1771..6554e156 100644 --- a/examples/gpmdk/src/gpmdcov_lib_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -38,7 +38,7 @@ module gpmdcov_lib_mod !subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& ! &charges_out,forces_out,verb_in) subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& - &charges_out,forces_out,dipole_out,bornch_out,verb_in) + &field_in,charges_out,forces_out,dipole_out,verb_in) ! Local modules @@ -71,9 +71,10 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors implicit none real(dp), allocatable, intent(in) :: coords_in(:,:), lattice_vectors_in(:,:) + real(dp), allocatable, intent(in) :: field_in(:) real(dp), allocatable, intent(inout) :: charges_out(:) real(dp), allocatable, intent(inout) :: forces_out(:,:) - real(dp), allocatable, intent(inout) :: bornch_out(:,:) + !real(dp), allocatable, intent(inout) :: bornch_out(:,:) real(dp), allocatable, intent(inout) :: dipole_out(:) integer, allocatable, intent(in) :: atomTypes_in(:) integer, intent(in) :: verb_in @@ -193,9 +194,13 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors write(*,*)"Dipole Moment=",dipoleMoment !call gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) - call gpmdcov_get_born_charges_v2(dipoleMoment,bornCharges,tb%norbi) - bornch_out(:,:) = bornCharges(:,:) - write(*,*)"Born F",bornCharges(:,1) + if(norm2(field_in) > 1.0D-10)then + call gpmdcov_get_field_forces(tb%norbi,field_in) + forces_out(:,:) = sy%force(:,:) + endif + !call gpmdcov_get_born_charges_v2(dipoleMoment,bornCharges,tb%norbi) + !bornch_out(:,:) = bornCharges(:,:) + !write(*,*)"Born F",bornCharges(:,1) if(lt%stopAt == "gpmdcov_Energ") then lib_init = .true. diff --git a/examples/gpmdk/src/gpmdcov_neighbor.F90 b/examples/gpmdk/src/gpmdcov_neighbor.F90 index 78b4a71d..0f69701f 100644 --- a/examples/gpmdk/src/gpmdcov_neighbor.F90 +++ b/examples/gpmdk/src/gpmdcov_neighbor.F90 @@ -55,6 +55,7 @@ module gpmdcov_neighbor_mod public :: gpmdcov_build_nlist_full, gpmdcov_destroy_nlist, gpmdcov_build_nlist_sparse public :: gpmdcov_build_nlist_sparse_v2, gpmdcov_build_nlist_sparse_v3, gpmd_nearestneighborlist,gpmdcov_get_vol + public :: gpmdcov_get_nlist_box_indices contains @@ -576,7 +577,140 @@ subroutine gpmdcov_get_vol(lattice_vectors,volBox) end subroutine gpmdcov_get_vol - + + + !! \brief It will bild a neighbor list using an "all to all" approach + !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. + !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. + !! \param nl Neighbor list type. + !! \param verbose Verbosity level. + !! \param rank MPI rank + subroutine gpmdcov_get_nlist_box_indices(coords,boxOfI,lattice_vectors,nx,ny,nz,verbose,rank,numranks) + implicit none + integer :: NBox, cnt, i, ibox + integer :: ith, ix, iy, iz + integer :: j, jbox, jj, jxBox + integer :: jyBox, jzBox, maxInBox, maxNeigh + integer :: myNumranks, myrank, nats, natsPerRank + integer, intent(in) :: nx, ny, nz + integer :: tx + integer :: ty, tz,nx1,ny1,nz1 + integer, allocatable :: inbox(:,:), ithFromXYZ(:,:,:) + integer, allocatable, intent(out) :: boxOfI(:) + integer, allocatable :: totPerBox(:), xBox(:), yBox(:), zBox(:) + integer, intent(in) :: verbose + integer, optional, intent(in) :: numranks, rank + real(dp) :: coordsNeigh(3), density, distance, translation(3) + real(dp) :: volBox, minx, miny, minz, smallReal, mlsnl + real(dp) :: maxx, maxy, maxz + real(dp), allocatable, intent(in) :: coords(:,:), lattice_vectors(:,:) + real(dp) :: rcutx,rcuty,rcutz +#ifdef DO_MPI + integer, allocatable :: rankRange(:,:) +#endif + + if(present(rank).and.present(numranks))then + myrank = rank + myNumranks = numranks + else + myrank = 1 + myNumranks = 1 + endif + + call gpmdcov_msI("gpmdcov_build_neigborlist","Building neighbor list ...",verbose,myrank) + + nats = size(coords,dim=2) !Get the number of atoms + natsPerRank = int(nats/myNumranks) + + !We will have approximatly [(4/3)*pi * rcut^3 * atomic density] number of neighbors. + !A very large atomic density could be 1 atom per (1.0 Ang)^3 = 1 atoms per Ang^3 + call gpmdcov_get_vol(lattice_vectors,volBox) + density = 1.0_dp + maxneigh = int(floor(3.14592_dp * (4.0_dp/3.0_dp) * density * (rcutx*rcuty*rcutz))) + + minx = 1.0d10 + miny = 1.0d10 + minz = 1.0d10 + maxx = -1.0d10 + maxy = -1.0d10 + maxz = -1.0d10 + do i = 1,nats + minx = min(minx,coords(1,i)) + miny = min(miny,coords(2,i)) + minz = min(minz,coords(3,i)) + maxx= max(maxx,coords(1,i)) + maxy = max(maxy,coords(2,i)) + maxz = max(maxz,coords(3,i)) + enddo + + !We assume the box is orthogona + rcutx = (maxx - minx)/(real(nx)) + rcuty = (maxy - miny)/(real(ny)) + rcutz = (maxz - minz)/(real(nz)) + + NBox = nx*ny*nz + maxInBox = int(density*(rcutx*rcuty*rcutz)) !Upper boud for the max number of atoms per box + mlsnl = mls() + allocate(inbox(NBox,maxInBox)) + inbox = 0 + allocate(totPerBox(Nbox)) + totPerBox = 0 + allocate(boxOfI(nats)) + boxOfI = 0 + allocate(xBox(Nbox)) + xBox = 0 + allocate(yBox(Nbox)) + yBox = 0 + allocate(zBox(Nbox)) + zBox = 0 + allocate(ithFromXYZ(nx,ny,nz)) + ithFromXYZ = 0 + + smallReal = -0.0000001_dp + !Search for the box coordinate and index of every atom + do i = 1,nats + !Index every atom respect to the discretized position on the simulation box. + !tranlation = coords(:,i) - origin !For the general case we need to make sure coords ar > 0 + ix = 1 + int(floor(abs((coords(1,i) - minx + smallReal)/(rcutx)))) !small box x-index of atom i + iy = 1 + int(floor(abs((coords(2,i) - miny + smallReal)/(rcuty)))) !small box y-index // + iz = 1 + int(floor(abs((coords(3,i) - minz + smallReal)/(rcutz)))) !small box z-index // + + write(*,*)coords(:,i),ix,iy,iz,(coords(3,i) - minz + smallReal)/rcutz + if(ix > nx .or. ix < 0)then + write(*,*)"ix",ix + Stop "Error in box index" + endif + if(iy > ny .or. iy < 0)then + write(*,*)"iy",iy + Stop "Error in box index" + endif + if(iz > nz .or. iz < 0)then + write(*,*)"iz",iz + Stop "Error in box index" + endif + + ith = ix + (iy-1)*(nx) + (iz-1)*(nx)*(ny) !Get small box index + boxOfI(i) = ith + + !From index to box coordinates + xBox(ith) = ix + yBox(ith) = iy + zBox(ith) = iz + + !From box coordinates to index + ithFromXYZ(ix,iy,iz) = ith + + totPerBox(ith) = totPerBox(ith) + 1 !How many per box + if(totPerBox(ith) > maxInBox) Stop "Exceeding the max in box allowed" + inbox(ith,totPerBox(ith)) = i !Who is in box ith + write(*,*)ix,iy,iz,boxOfI(i),coords(:,i) + + enddo + + + end subroutine gpmdcov_get_nlist_box_indices + + !! \brief It will bild a neighbor list using an "all to all" approach !! \param coords System coordinates. coords(1,7): x-coordinate of atom 7. !! \param lattice_vectors. Lattice vectors of the system box. lattice_vectors(1,3): z-coordinate of vector 1. diff --git a/examples/gpmdk/src/gpmdcov_vars.F90 b/examples/gpmdk/src/gpmdcov_vars.F90 index 9a9a169f..1544f1dc 100644 --- a/examples/gpmdk/src/gpmdcov_vars.F90 +++ b/examples/gpmdk/src/gpmdcov_vars.F90 @@ -62,6 +62,7 @@ module gpmdcov_vars integer, allocatable :: xadj(:), adjncy(:), CH_count(:), norbsInEachCH(:) integer, allocatable :: part(:), core_count(:), Halo_count(:,:) integer, allocatable :: partsInEachRank(:), reshuffle(:,:), npartsVect(:), displ(:), PartsInRankI(:) + integer, allocatable :: whichParts_guess_saved(:) real(dp) :: C0, C1, C2, C3 real(dp) :: C4, C5, ECoul, ECoulU, ECoulK, ECoulR, EKIN, beta, kbt real(dp) :: EPOT, ERep, Energy, Etot, nocc diff --git a/src/prg_graph_mod.F90 b/src/prg_graph_mod.F90 index c032188d..fe31445e 100644 --- a/src/prg_graph_mod.F90 +++ b/src/prg_graph_mod.F90 @@ -26,6 +26,7 @@ module prg_graph_mod public :: prg_equalGroupPartition public :: prg_filePartition public :: prg_fnormGraph + public :: prg_sedacsPartition !> Subgraph type type subgraph_t @@ -101,6 +102,11 @@ module prg_graph_mod !> Total number of local partitions integer :: nparts + !> For partitioning in x,y,z direction + !! This is only used in case of "domain" type + !! of partitioning + integer :: nx, ny, nz + !> Number of nodes in each local partition integer, allocatable :: nnodesInPart(:) @@ -345,6 +351,7 @@ subroutine prg_printGraphPartitioning(gp) end subroutine prg_printGraphPartitioning + !> Create equal graph partitions, based on number of rows/orbitals !! \param gp Graph partitioning` !! \param nodesPerPart Number of core nodes per partition @@ -367,8 +374,8 @@ subroutine prg_equalPartition(gp, nodesPerPart, nnodes) call prg_destroyGraphPartitioning(gp) call prg_initGraphPartitioning(gp, pname, np, nnodes, nnodes) - !! Assign node ids (mapped to orbitals as rows) to each node in each - !! partition + ! Assign node ids (mapped to orbitals as rows) to each node in each + ! partition !$omp parallel do default(none) private(i) & !$omp private(it,j,psize) & !$omp shared(gp,nnodes,nodesPerPart) @@ -391,6 +398,417 @@ subroutine prg_equalPartition(gp, nodesPerPart, nnodes) end subroutine prg_equalPartition + + !> Create a partitioning based on node flips (as implemented in SEDACS - with several changes) + !! \param gp Graph partitioning type + !! \param whichParts_guess_saved Array indicating the "color" a node belongs to + !! \param g_bml Adjacency matrix (BML type) + !! \param nparts Number of total parts + !! \param nnodes Total nodes in the graph (typically consistent with number of atoms) + !! \param verb Verbosity level + !! + subroutine prg_sedacsPartition(gp,coords,whichParts_guess_saved,g_bml,nparts,nnodes,verb) + implicit none + type (graph_partitioning_t), intent(inout) :: gp + integer, intent(in) :: nnodes, nparts + integer, allocatable, intent(inout) :: whichParts_guess_saved(:) + integer, optional, intent(in) :: verb + integer :: i, cnt, part, j, it, psize, cut, cutOld, ac, nac + integer :: mdim, nodesPerPart,toBeDistributed,sumSizes,rem + integer, allocatable :: whichParts(:) + integer, allocatable :: graph(:,:),degs(:) + real(kind(1.0)), allocatable :: row(:) + real(dp) :: bal,sumDegs + real(dp), allocatable, intent (in) :: coords(:,:) + character(len=100) :: pname + type (bml_matrix_t), intent(in) :: g_bml + logical :: writeOut + + if(present(verb))then + if(verb >= 1)then + write(*,*)"Doing SEDACS type of partitioning ..." + if(verb >= 2) writeOut = .true. + endif + endif + + !If there is no guess we will do a "chunk/block" partitioning + !This initial pratitioning has all the nodes evenly distributed + if(.not. allocated(whichParts_guess_saved))then + nodesPerPart = int(nnodes/nparts) + rem = nnodes - nodesPerPart*nparts + toBeDistributed = rem + + pname = '("equalParts")' + call prg_destroyGraphPartitioning(gp) + call prg_initGraphPartitioning(gp, pname, nparts, nnodes, nnodes) + + sumSizes = 0 + do i = 1, gp%totalParts + call prg_initSubgraph(gp%sgraph(i), i, nnodes) + if (sumSizes .le. nnodes - nodesPerPart) then + psize = nodesPerPart + min(toBeDistributed,1) + toBeDistributed = max(toBeDistributed - 1,0) + else + psize = nnodes - sumSizes + endif + write(*,*)i,psize,sumSizes + allocate(gp%sgraph(i)%nodeInPart(psize)) + do j = 1, psize + it = sumSizes + j-1; + gp%sgraph(i)%nodeInPart(j) = it + enddo + sumSizes = sumSizes + psize + gp%nnodesInPart(i) = psize + gp%nnodesInPartAll(i) = psize + enddo + + call prg_get_parts_indices(gp,nnodes,whichParts) + allocate(whichParts_guess_saved(nnodes)) + else + allocate(whichParts(nnodes)) + whichParts = whichParts_guess_saved + endif + + !Get degrees and build 2D-sedacs-like graph + !In this case the degreess are saved in a separate array + mdim = bml_get_m(g_bml) + allocate(graph(nnodes,mdim)) + allocate(degs(nnodes)) + allocate(row(nnodes)) + row = 0.0 + degs = 0 + do i = 1,nnodes + call bml_get_row(g_bml,i,row) + do j = 1,nnodes + if(row(j) .ge. 0.5)then + degs(i) = degs(i) + 1 + graph(i,degs(i)) = j + endif + enddo + enddo + deallocate(row) + + sumDegs = sum(degs) + call prg_get_graphCut(whichParts,degs,graph,cut) !Get the initial cut + if(writeOut) write(*,*)"Iter, Cut, RelCut, and Bal",0,cut, cut/sumDegs,bal + cutOld = cut + do i = 1,200 + call do_flips_precomp(whichParts,coords,graph,g_bml,degs,nnodes,nparts,bal) + call prg_get_graphCut(whichParts,degs,graph,cut) + call prg_get_balancing(whichParts,nparts,bal) + if(writeOut) write(*,*)"Iter, Cut, RelCut, and Bal",i,cut, cut/sumDegs,bal + if(cut == cutOld)then + exit + else + cutOld = cut + endif + enddo + + deallocate(graph) + deallocate(degs) + + pname = '("sedacsPartition")' + call prg_destroyGraphPartitioning(gp) + call prg_initGraphPartitioning(gp, pname, nparts, nnodes, nnodes) + + !Assign node ids to each node in each part + !$omp parallel do default(none) private(i) & + !$omp private(cnt,it,j,psize,part) & + !$omp shared(gp,nnodes,whichParts) + do i = 1, gp%totalParts + call prg_initSubgraph(gp%sgraph(i), i, nnodes) + psize = int(sum(whichParts, mask=(whichParts==i))/i) + allocate(gp%sgraph(i)%nodeInPart(psize)) + cnt = 0 + do j = 1,nnodes + part = whichParts(j) + if(part == i)then + cnt = cnt + 1 + gp%sgraph(i)%nodeInPart(cnt) = j - 1 + endif + enddo + gp%nnodesInPart(i) = psize + gp%nnodesInPartAll(i) = psize + enddo + !$omp end parallel do + + whichParts_guess_saved = whichParts + deallocate(whichParts) + + end subroutine prg_sedacsPartition + + + !! Do node partition flips with precomputed cuts. + !! \brief This function is a special case of do_flips where + !! the cuts around a node are precomputed for all possible + !! part index that same node could have. This will differ from the do_flips + !! since everytime there is a flip, there is no actualization of the cuts. The + !! price to pay is the need of more iterations until convergence. + !! \param whichPart partition indexing vector. + !! \param graph Graph to be partition. graph[i,0] = degree of node i. + !! graph[i,j>0] = the node conected to node i. + !! \param nnodes Number of nodes. + !! \param nparts Number of parts. + !! \return whichPartNew New partition indexing verctor. + ! + subroutine do_flips_precomp(whichParts,coords,graph,adj,degs,nnodes,nparts,bal) + implicit none + integer, allocatable :: cutsI(:,:) + integer, intent(in) :: nnodes,nparts + integer, allocatable, intent(inout) :: graph(:,:),degs(:) + integer, allocatable, intent(inout) :: whichParts(:) + real(dp) :: bal + real(kind(1.0)), allocatable :: row(:) + type(bml_matrix_t) :: adj + integer :: i,j,deg,ind,ii,origCut,newCut, sumDeltaCut,ac,nac,newCut1 + integer :: origCutI,newCutI,origCutJ,newCutJ + integer :: partIndexI, partIndexII,partIndexJ,k,cut,cutOld,newPartIndex,inDefect,discon + integer, allocatable :: momentI(:),momentsI(:,:),partSizes(:) + real(dp), allocatable, intent (in) :: coords(:,:) + logical :: disconnected, conditionToFlip + + + allocate(cutsI(nnodes,nparts)) + cutsI = 0 + sumDeltaCut = 0 + ac = 0 + nac = 0 + inDefect = 0 + + !$omp parallel do default(none) private(i) & + !$omp private(ii,deg,partIndexII,ind) & + !$omp shared(degs,graph,nnodes,whichParts,cutsI) + do i = 1,nnodes + deg = degs(i) + !Get the max cut a node could have + cutsI(i,:) = deg + !Lets look at every neighbor + do ii = 1,deg + ind = graph(i,ii) + partIndexII = whichParts(ind) + !Everytime there is a neighbor in a certain part + !it will decrese the cut of I if I would be on that + !same part. + cutsI(i,partIndexII) = cutsI(i,partIndexII) - 1 + enddo + enddo + !$omp end parallel do + + !Get sizes + allocate(row(nnodes)) + allocate(partSizes(nparts)) + partSizes = 0 + + do i = 1,nnodes + partIndexI = whichParts(i) + partSizes(partIndexI) = partSizes(partIndexI) + 1 + enddo + + do i = 1,nnodes + call bml_get_row(adj,i,row) + if(cutsI(i,whichParts(i)) > 0)then + do j = i+1, nnodes + if((whichParts(i) .ne. whichParts(j)))then + !conditionToFlip = .false. + !disconnected = .false. + + !Look at their neighbors and count the cuts + partIndexI = whichParts(i) + partIndexJ = whichParts(j) + origCutI = cutsI(i,partIndexI) + origCutJ = cutsI(j,partIndexJ) + + newCutI = cutsI(i,partIndexJ) + + if(newCutI < degs(i))then + newCutJ = cutsI(j,partIndexI) + if(newCutJ < degs(j))then + + !Now we know the cut when I is in partIndexI and J + origCut = origCutI + origCutJ + newCut = newCutI + newCutJ + + if((newCut <= origCut))then + + ac = ac + 1 + whichParts(i) = partIndexJ + whichParts(j) = partIndexI + + !Actualizing possible cuts of neighbors of i and j + do ii = 1,degs(i) + ind = graph(i,ii) + !The nighs of i "if now they are in the "new color + !of i", their cut "at that color" will be decreased + !by one. + cutsI(ind,partIndexJ) = cutsI(ind,partIndexJ) - 1 + cutsI(ind,partIndexI) = cutsI(ind,partIndexI) + 1 + enddo + do ii = 1,degs(j) + ind = graph(j,ii) + !Same for the neighs of j + cutsI(ind,partIndexI) = cutsI(ind,partIndexI) - 1 + cutsI(ind,partIndexJ) = cutsI(ind,partIndexJ) + 1 + enddo + endif + endif + else + nac = nac + 1 + endif + + elseif(whichParts(i) .eq. whichParts(j))then + if(row(j) < 0.5)then + origCut = 0 + newCut = 0 + partIndexI = whichParts(i) + partIndexJ = whichParts(j) + + origCut = cutsI(i,partIndexI) + origCut = origCut + cutsI(j,partIndexJ) + + ! Change color of I + inDefect = minloc(partSizes,dim=1) + + newPartIndex = inDefect + + newCut = cutsI(i,newPartIndex) + newCut = newCut + cutsI(j,partIndexJ) + + newCut1 = cutsI(i,partIndexI) + newCut1 = newCut1 + cutsI(j,newPartIndex) + + if(newCut < origCut)then + whichParts(i) = newPartIndex + whichParts(j) = partIndexJ + if(newPartIndex .ne.partIndexJ)then + partSizes(newPartIndex) = partSizes(newPartIndex) + 1 + partSizes(partIndexJ) = partSizes(partIndexJ) - 1 + endif + !sumDeltaCut = newCut - origCut + sumDeltaCut + elseif(newCut1 < origCut)then + whichParts(i) = partIndexI + whichParts(j) = newPartIndex + if(newPartIndex .ne. partIndexI)then + partSizes(newPartIndex) = partSizes(newPartIndex) + 1 + partSizes(partIndexJ) = partSizes(partIndexJ) - 1 + endif + !sumDeltaCut = newCut - origCut + sumDeltaCut + else + whichParts(i) = partIndexI + whichParts(j) = partIndexJ + endif + endif + else + cycle + endif + enddo + endif + enddo + + deallocate(partSizes) + deallocate(cutsI) + deallocate(row) + + end subroutine do_flips_precomp + + + + !> Get balancing + !! \param whichParts Vector containing the "color" (part) + !! of each node. + !! \param np Total number of parts + !! \param bal Computed balancing + !! + subroutine prg_get_balancing(whichParts,np,bal) + implicit none + integer, allocatable, intent(in) :: whichParts(:) + integer, allocatable :: partsSizes(:) + integer :: np, i, nnodes + real(dp), intent(out) :: bal + + allocate(partsSizes(np)) + nnodes = size(whichParts) + partsSizes = 0 + do i = 1,nnodes + !write(*,*)"whichParts",i,nnodes,whichParts(i),np + if(whichParts(i) > np)then + write(*,*)"!!!ERROR: Part index",whichParts(i),& + "is larger than the total number of parts,",np + stop + endif + partsSizes(whichParts(i)) = partsSizes(whichParts(i)) + 1 + enddo + bal = real(maxval(partsSizes))/real(minval(partsSizes)) + deallocate(partsSizes) + end subroutine prg_get_balancing + + !> Get part indices + !! \param gp Graph partitioning type + !! \param nnods Number of nodes + !! \param whichParts Vector containing the "color" (part) + !! of each node. + !! + subroutine prg_get_parts_indices(gp,nnodes,whichParts) + type (graph_partitioning_t), intent(in) :: gp + integer, allocatable,intent(out) :: whichParts(:) + integer, intent(in) :: nnodes + integer :: i,j,k,kk,partIndex + + if(.not. allocated(whichParts)) allocate(whichParts(nnodes)) + + partIndex = 0 + whichParts = 0 + + !$omp parallel do default(none) private(i) & + !$omp private(k,kk) & + !$omp shared(gp,nnodes,whichParts) + do i = 1,gp%totalParts !Loop over parts + do j = 1,gp%nnodesInPart(i) !Loop over nodes in part + k = gp%sgraph(i)%nodeInPart(j) + kk = k + 1 !Indexig starts from 0 for all sgraphs (beacuse of historical reasons) + whichParts(kk) = i + enddo + enddo + !$omp end parallel do + + end subroutine prg_get_parts_indices + + + !> Get graph cut + !! \brief Get the total cut (edges crossing parts) + !! \param whichParts Vector containing the "color" (part) + !! of each node. + !! \param degs Degrees of every node + !! \param graph 2D array containing the information of every + !! connection. graph(i,j): index of the jth neigbor of i. + !! \param cut Computed cut + !! + subroutine prg_get_graphCut(whichParts,degs,graph,cut) + integer, allocatable, intent(in) :: whichParts(:) + integer, intent(out) :: cut + integer :: nnodes,i,j,partIndexI,partIndexJ,k,kk + integer :: ind + integer, allocatable, intent(in) :: graph(:,:),degs(:) + + cut = 0 + nnodes = size(whichParts) + + do i = 1,nnodes + !write(*,*)"graph",graph(i,1:degs(i)) + partIndexI = whichParts(i) + !Look at the neighbors and check if they are in different part + do j = 1,degs(i) + ind = graph(i,j) + partIndexJ = whichParts(ind) + if(int(partIndexI - partIndexJ) .ne. 0)then + cut = cut + 1 + endif + enddo + enddo + cut = 0.5_dp*cut !Because of the double counting + + end subroutine prg_get_graphCut + + !> Create equal group graph partitions, based on number of atoms/groups !! \param gp Graph partitioning !! \param hindex Node indeces that represent ranges of atoms/groups diff --git a/src/prg_graphsp2parser_mod.F90 b/src/prg_graphsp2parser_mod.F90 index fcfdb895..275af7d3 100644 --- a/src/prg_graphsp2parser_mod.F90 +++ b/src/prg_graphsp2parser_mod.F90 @@ -32,6 +32,12 @@ module prg_graphsp2parser_mod integer :: nodesPerPart integer :: natoms integer :: partition_count + + !> For partitioning in x,y,z direction + !! This is only used in case of "domain/spatial" type + !! of partitioning + integer :: nx, ny, nz + real(dp) :: sp2tol real(dp) :: threshold real(dp) :: bndfil @@ -62,7 +68,7 @@ subroutine prg_parse_gsp2(gsp2data,filename) implicit none type(gsp2data_type), intent(inout) :: gsp2data - integer, parameter :: nkey_char = 7, nkey_int = 8, nkey_re = 7, nkey_log = 2 + integer, parameter :: nkey_char = 7, nkey_int = 11, nkey_re = 7, nkey_log = 2 character(len=*) :: filename !Library of keywords with the respective defaults. @@ -74,9 +80,10 @@ subroutine prg_parse_gsp2(gsp2data,filename) character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & 'Mdim=', 'MinSP2Iter=', 'MaxSP2Iter=','Ndim=', 'NodesPerPart=', 'NAtoms=', & - 'PartitionCount=', 'PartEach='] + 'PartitionCount=', 'PartEach=', 'PartitionCountX=','PartitionCountY=',& + &'PartitionCountZ='] integer :: valvector_int(nkey_int) = (/ & - -1, 10, 100, 1, 16, 1, 1,1 /) + -1, 10, 100, 1, 16, 1, 1,1,0,0,0 /) character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & 'MatrixThreshold=','SP2Tol=','BndFil=', 'GraphThreshold=', 'ErrLimit=', 'CovGraphFact=', 'NLGraphCut=' ] @@ -134,6 +141,9 @@ subroutine prg_parse_gsp2(gsp2data,filename) gsp2data%natoms = valvector_int(6) gsp2data%partition_count= valvector_int(7) gsp2data%parteach= valvector_int(8) + gsp2data%nx = valvector_int(9) + gsp2data%ny = valvector_int(10) + gsp2data%nz = valvector_int(11) end subroutine prg_parse_gsp2 diff --git a/src/prg_system_mod.F90 b/src/prg_system_mod.F90 index 1e92f56b..0168f5fb 100644 --- a/src/prg_system_mod.F90 +++ b/src/prg_system_mod.F90 @@ -1892,6 +1892,7 @@ subroutine prg_get_covgraph(sy,nnStruct,nrnnstruct,bml_type,factor,gcov_bml,mdim type(bml_matrix_t), intent(inout) :: gcov_bml type(system_type), intent(in) :: sy logical(1), allocatable :: ispresent(:) + real(kind(1.0)), allocatable :: row(:) if(bml_get_N(gcov_bml).gt.0) call bml_deallocate(gcov_bml) From a61c7a6ae7d8b1c73702f246ac5f9cf4e3db4158 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Tue, 16 Jul 2024 22:46:15 -0600 Subject: [PATCH 12/69] Added main fro field --- .../run_lib/main_field_induced_forces.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 examples/gpmdk/run_lib/main_field_induced_forces.py diff --git a/examples/gpmdk/run_lib/main_field_induced_forces.py b/examples/gpmdk/run_lib/main_field_induced_forces.py new file mode 100755 index 00000000..ee8d48b3 --- /dev/null +++ b/examples/gpmdk/run_lib/main_field_induced_forces.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import numpy as np +from sdc_system import * +from sdc_ptable import ptable +import ctypes as ct +import os +import scipy.linalg as sp +from gpmd import * + + +for i in range(1): + #GPMD verbosity level. + verb = 0 + #Load the coordinates and atom types + #latticeVectors,symbols,atomTypes,coords = read_xyz_file("H2.xyz",lib="None",verb=True) + latticeVectors,symbols,atomTypes,coords = read_pdb_file("benzene-CN.pdb",lib="None",verb=True) + #latticeVectors,symbols,atomTypes,coords = read_pdb_file("H2O.pdb",lib="None",verb=True) + print("Processing molecule H2O") + + #Call the general gpmd funtion + field = np.zeros((3)) + field[0] = 0.0 ; field[1] = 0.0 ; field[2] = 0.0 + err,charges_out,forces_out,dipole_out = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + + print("Charge on atom 1",charges_out[0]) + print("Dipole",dipole_out) + print("Forces on atom 1",forces_out[0,:]) + + field[0] = 0.01 ; field[1] = 0.0 ; field[2] = 0.0 + err,charges_out,forces_out,dipole_out = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + print("Forces on atom 1",forces_out[0,:]) + + + # tch = 0.0 + # bchMat = np.zeros((3,3)) + # for i in range(len(bornch_out[:,0])): + # bchMat[0,:] = bornch_out[i,0:3] + # bchMat[1,:] = bornch_out[i,3:6] + # bchMat[2,:] = bornch_out[i,6:9] + # E,Q = sp.eigh(bchMat) + # print(i,"\nbc",bchMat) + # print("E",E) + # print("resultant charge",np.sum(E)) + # print("Muliken charge",charges_out[i]) + + From f62b952ab505ab4ca5c805ad9491148924a94df9 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 15 Jul 2024 09:07:50 -0600 Subject: [PATCH 13/69] NVTX tags --- examples/gpmdk/src/CMakeLists.txt | 9 +++++++-- examples/gpmdk/src/gpmdcov_mdloop.F90 | 12 +++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/examples/gpmdk/src/CMakeLists.txt b/examples/gpmdk/src/CMakeLists.txt index dfaf40e1..f9293dbc 100644 --- a/examples/gpmdk/src/CMakeLists.txt +++ b/examples/gpmdk/src/CMakeLists.txt @@ -164,6 +164,7 @@ list( "gpmdcov_constraints.F90" "gpmdcov_highorder.F90" "gpmdcov_field.F90" +"gpmdcov_nvtx.F90" ) @@ -192,14 +193,18 @@ target_link_libraries(${myappendix} PUBLIC LINK_FLAGS "") endfunction(progress_appendix_library) +if(GPMDK_NVTX) + add_definitions(-DUSE_NVTX) +endif() + if(LIB) - progress_appendix_library( gpmd gpmdcov_lib.F90 gpmdcov_lib_mod.F90 ) + progress_appendix_library( gpmdk gpmdcov_lib.F90 gpmdcov_lib_mod.F90 ) #progress_appendix( gpmd_a gpmd_secuential.F90 gpmdcov_lib_mod.F90) file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/gpmd.py DESTINATION ${PROJECT_BINARY_DIR}) else() progress_appendix( gpmdk gpmdk.F90 ) - progress_appendix( gpmd_a gpmd_secuential.F90 gpmdcov_lib_mod.F90) + #progress_appendix( gpmd_a gpmd_secuential.F90 gpmdcov_lib_mod.F90) endif() SET(DESTINATION ${PROJECT_BINARY_DIR}) diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 index aede9769..e6d01850 100644 --- a/examples/gpmdk/src/gpmdcov_mdloop.F90 +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -17,6 +17,10 @@ subroutine gpmdcov_MDloop() use gpmdcov_langevin_mod use gpmdcov_preparemd_mod +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif + real(dp) :: mls_md, mls_md1, mls_md2, resnorm real(dp), allocatable :: kernelTimesRes(:), n1(:) real(dp), allocatable :: KK0Res(:) @@ -69,6 +73,9 @@ subroutine gpmdcov_MDloop() mls_md = mls() +#ifdef USE_NVTX + call nvtxStartRange("MD iter",1) +#endif mls_md1 = mls() if(myRank == 1)then @@ -519,7 +526,10 @@ subroutine gpmdcov_MDloop() call gpmdcov_msI("gpmdcov_MDloop","Time for MD iter & &"//to_string(mls() - mls_md)//" ms",lt%verbose,myRank) - +#ifdef USE_NVTX + call nvtxEndRange +#endif + ! Save MD state each 120 steps if(gpmdt%dumpeach .gt. 0)then if(mod(mdstep,gpmdt%dumpeach) == 0)call gpmdcov_dump() From c1aaa69b349cf4e46e5e95246dcefcdc3eb24b49 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 17 Jul 2024 11:02:03 -0600 Subject: [PATCH 14/69] Introducing nvtx tags and some optimizations --- examples/gpmdk/build_venado_hackathon.sh | 4 +- examples/gpmdk/run/TrpCage/input.in | 8 +- examples/gpmdk/run/TrpCage/sbatch_profile.sh | 4 +- examples/gpmdk/run/TrpCage/wrapper_profile.sh | 6 +- examples/gpmdk/run/water/input.in | 2 +- examples/gpmdk/run/water/sbatch.sh | 6 +- examples/gpmdk/src/gpmdcov_allocation.F90 | 24 ++++- examples/gpmdk/src/gpmdcov_kernel.F90 | 65 ++++++++++--- examples/gpmdk/src/gpmdcov_mdloop.F90 | 93 +++++++++++++++++-- examples/gpmdk/src/gpmdcov_neighbor.F90 | 2 + 10 files changed, 176 insertions(+), 38 deletions(-) diff --git a/examples/gpmdk/build_venado_hackathon.sh b/examples/gpmdk/build_venado_hackathon.sh index 465d4b24..0aeb42e6 100755 --- a/examples/gpmdk/build_venado_hackathon.sh +++ b/examples/gpmdk/build_venado_hackathon.sh @@ -2,9 +2,9 @@ WORKDIR=/usr/projects/icapt/mewall/venado/packages rm -rf build_hackathon mkdir build_hackathon cd build_hackathon -cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" \ +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" \ -DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ #-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ -make +make -j1 diff --git a/examples/gpmdk/run/TrpCage/input.in b/examples/gpmdk/run/TrpCage/input.in index 165ce61f..b05d89c7 100644 --- a/examples/gpmdk/run/TrpCage/input.in +++ b/examples/gpmdk/run/TrpCage/input.in @@ -31,7 +31,7 @@ Latte{ TimeRatio= 10.0 TimeStep= 0.25 #TimeStep= 0.00 - MDSteps= 1000 + MDSteps= 20 #ParamPath= "../sulfurTBparam" ParamPath= "../../tests/latteTBparams" #ParamPath= "../latteTBparams_orig" @@ -76,7 +76,7 @@ Latte{ #CoordsFile= Mac1_latte.dat #CoordsFile= ICH.dat CoordsFile= input.dat - NlistEach= 1 + NlistEach= 10 #BMLDistributionType= Distributed MuCalcType= FromParts #MuCalcType= Combined @@ -120,7 +120,7 @@ GSP2{ NLGraphCut= 4.5 CovGraphFact= 4.5 #CovGraphFact= 3.0 - #PartitionCount= 128 + PartitionCount= 256 #PartitionCount= 8 #PartitionCount= 512 #PartitionCount= 16 @@ -131,7 +131,7 @@ GSP2{ #PartitionCount= 1600 #PartitionCount= 32 #PartitionCount= 64 - PartitionCount= 256 + #PartitionCount= 32 #PartitionCount= 3200 #PartitionCount= 6400 #PartitionCount= 1 diff --git a/examples/gpmdk/run/TrpCage/sbatch_profile.sh b/examples/gpmdk/run/TrpCage/sbatch_profile.sh index 3ebf58e8..77ac0576 100755 --- a/examples/gpmdk/run/TrpCage/sbatch_profile.sh +++ b/examples/gpmdk/run/TrpCage/sbatch_profile.sh @@ -4,7 +4,7 @@ ##SBATCH --reservation=gpu_debug ##SBATCH --qos=debug ##SBATCH --time 2:00:00 -#SBATCH -N 32 +#SBATCH -N 64 ##SBATCH -A ichelp_g ##SBATCH -A w23_macroqmd_g @@ -14,5 +14,5 @@ source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_ve #export MPICH_ALLREDUCE_NO_SMP=1 export MPICH_SMP_SINGLE_COPY_MODE=NONE -OMP_NUM_THREADS=72 srun -n 128 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper_profile.sh +OMP_NUM_THREADS=72 srun -n 256 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper_profile.sh diff --git a/examples/gpmdk/run/TrpCage/wrapper_profile.sh b/examples/gpmdk/run/TrpCage/wrapper_profile.sh index 6c4fb53c..e380fb4d 100644 --- a/examples/gpmdk/run/TrpCage/wrapper_profile.sh +++ b/examples/gpmdk/run/TrpCage/wrapper_profile.sh @@ -6,7 +6,11 @@ export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) RUN="/usr/projects/icapt/mewall/venado/packages/qmd-progress/examples/gpmdk/build_hackathon/gpmdk" #stdbuf -o0 $RUN input_ICH.in |& tee out_ICH if [ $SLURM_PROCID -eq 0 ]; then - nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -t mpi,openmp,cuda,nvtx --mpi-impl=mpich $RUN input.in >& out_0 +# nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -t mpi,openmp,cuda,nvtx --mpi-impl=mpich --delay=90 $RUN input.in >& out_0 + nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -c cudaProfilerApi -t mpi,openmp,cuda,nvtx --mpi-impl=mpich -o report_%q{SLURM_PROCID} $RUN input.in >& out_$SLURM_PROCID + #nsys profile -t mpi,openmp,cuda,nvtx --mpi-impl=mpich --delay=90 -o report_%q{SLURM_PROCID} $RUN input.in >& out_$SLURM_PROCID else $RUN input.in >& /dev/null + #nsys profile --start-later -t mpi,openmp,cuda,nvtx --mpi-impl=mpich -o report_%q{SLURM_PROCID} $RUN input.in >& /dev/null + #nsys profile -t mpi,openmp,cuda,nvtx --mpi-impl=mpich --delay=90 -o report_%q{SLURM_PROCID} $RUN input.in >& /dev/null fi diff --git a/examples/gpmdk/run/water/input.in b/examples/gpmdk/run/water/input.in index 014a48ce..7d1f61b5 100644 --- a/examples/gpmdk/run/water/input.in +++ b/examples/gpmdk/run/water/input.in @@ -31,7 +31,7 @@ Latte{ TimeRatio= 10.0 TimeStep= 0.2 #TimeStep= 0.00 - MDSteps= 1000 + MDSteps= 20 #ParamPath= "../sulfurTBparam" ParamPath= "../../tests/latteTBparams" #ParamPath= "../latteTBparams_orig" diff --git a/examples/gpmdk/run/water/sbatch.sh b/examples/gpmdk/run/water/sbatch.sh index 166b6caf..ca5ee53c 100755 --- a/examples/gpmdk/run/water/sbatch.sh +++ b/examples/gpmdk/run/water/sbatch.sh @@ -3,19 +3,17 @@ ##SBATCH -p gpu_debug ##SBATCH --reservation=gpu_debug ##SBATCH --qos=debug -##SBATCH --time 16:00:00 -##SBATCH --time 2:00:00 +#SBATCH --time 2:00:00 #SBATCH -N 4 ##SBATCH -A ichelp_g ##SBATCH -A w23_macroqmd_g cd ${PWD} - source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh #export MPICH_ALLREDUCE_NO_SMP=1 export MPICH_SMP_SINGLE_COPY_MODE=NONE +#export MPICH_OPT_THREAD_SYNC=0 -#OMP_NUM_THREADS=32 srun -n 8 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh OMP_NUM_THREADS=72 srun -n 16 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh diff --git a/examples/gpmdk/src/gpmdcov_allocation.F90 b/examples/gpmdk/src/gpmdcov_allocation.F90 index 5a4138b0..1f9caf72 100644 --- a/examples/gpmdk/src/gpmdcov_allocation.F90 +++ b/examples/gpmdk/src/gpmdcov_allocation.F90 @@ -15,9 +15,28 @@ module gpmdcov_allocation_mod public :: gpmdcov_reallocate_denseBmlRealMat public :: gpmdcov_vect2MatInt public :: gpmdcov_mat2VectInt - + public :: gpmdcov_bml_set_N + public :: gpmdcov_bml_allocated + contains + function gpmdcov_bml_allocated(myMat) + implicit none + type(bml_matrix_t) :: myMat + logical :: gpmdcov_bml_allocated + + gpmdcov_bml_allocated = bml_allocated(myMat) + end function gpmdcov_bml_allocated + + subroutine gpmdcov_bml_set_N(myMat,mySize) + implicit none + integer :: mySize + type(bml_matrix_t) :: myMat + + call bml_set_N_dense(myMat,mySize) + + end subroutine gpmdcov_bml_set_N + !> To reallocate a vector !! subroutine gpmdcov_reallocate_realVect(myVect,mySize) @@ -87,7 +106,8 @@ subroutine gpmdcov_reallocate_denseBmlRealMat(myMat,mySize) if(bml_allocated(myMat))then call bml_deallocate(myMat) - endif + endif + call bml_zero_matrix("dense",bml_element_real,dp,mySize,mySize,myMat) end subroutine gpmdcov_reallocate_denseBmlRealMat diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 0ed8066b..4adf65be 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -11,6 +11,10 @@ module gpmdcov_kernel_mod use gpmdcov_allocation_mod use gpmdcov_writeout_mod +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif + implicit none private @@ -796,9 +800,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re real(dp), intent(inout), allocatable :: KK0Res(:) real(dp) :: trP1(1), trdPdMu(1) real(dp) :: mu1_global, error - type(bml_matrix_t) :: ptham_bml, ptaux_bml - type(bml_matrix_t) :: zq_bml,zqt_bml - type(bml_matrix_t) :: p1_bml,dPdMu_bml + type(bml_matrix_t),save :: ptham_bml, ptaux_bml + type(bml_matrix_t),save :: zq_bml,zqt_bml + type(bml_matrix_t),save :: p1_bml,dPdMu_bml type(system_type), allocatable, intent(inout) :: mysyprt(:) type(system_type), allocatable, intent(inout) :: mysyprtk(:) real(dp) :: mynumel(10), mls_v @@ -902,6 +906,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re c_i = 0.0_dp allocate(ff(maxCoresAmongPartsAndRanks,partsInEachRank(myRank),mRanks)) ff = 0.0_dp + !Here we enter the loop for the rank updates (do not confuse with MPI rank) do irank = 1, mRanks @@ -914,7 +919,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re vi(:,irank) = vi(:,irank)/norm2(vi(:,irank)) endif chargePertVect = vi(:,irank) - +#ifdef USE_NVTX + call nvtxStartRange("Ewald",1) +#endif call get_ewald_list_real_dcalc_vect(sy%spindex,sy%splist,sy%coordinate& ,chargePertVect,tb%hubbardu,sy%lattice_vector,& sy%volr,lt%coul_acc,lt%timeratio,nl%nnIx,nl%nnIy,& @@ -923,7 +930,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call get_ewald_recip(sy%spindex,sy%splist,sy%coordinate& ,chargePertVect,tb%hubbardu,sy%lattice_vector,& sy%recip_vector,sy%volr,lt%coul_acc,my_coul_forces_k,my_coul_pot_k); - +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_reallocate_realMat(q1,maxCoresAmongPartsAndRanks,partsInEachRank(myRank)) call gpmdcov_reallocate_realMat(dqdmu,maxCoresAmongPartsAndRanks,partsInEachRank(myRank)) @@ -964,12 +973,27 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 v_core_i(myj,iptt,irank) = vi(myjj,irank) enddo - +#ifdef USE_NVTX + call nvtxStartRange("BML_math",2) +#endif !COMMENT: MAYBE v_core_i should not be normalized!!!!!! !Computing the perturbative Hamiltonian (H1) from the coulombic perturbations !generated by dr=K0*res. At the C+H level - call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) +#ifdef USE_NVTX + call nvtxStartRange("Reallocation event",7) +#endif + if (.not.bml_allocated(ptham_bml))then + call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,8000) + else + write(*,*)"ptham_bml already allocated, using bml_set_N" + call gpmdcov_bml_set_N(ptham_bml,norbs) + endif + +! call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_reallocate_denseBmlRealMat(ptaux_bml,norbs) !ptaux_bml corresponds to H0 which is 0 in this case. call prg_get_hscf_v2(ptaux_bml,mysyprt(ipt)%estr%over,ptham_bml,mysyprt(ipt)%spindex,& @@ -992,16 +1016,26 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call gpmdcov_reallocate_realVect(dPdMu,norbs) norbsCore = mysyprt(ipt)%estr%norbsCore call gpmdcov_reallocate_denseBmlRealMat(p1_bml,norbs) - +#ifdef USE_NVTX + call nvtxEndRange +#endif !call prg_canon_response2_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& ! &mysyprt(ipt)%estr%evals,ef,12,norbs,lt%threshold) +#ifdef USE_NVTX + call nvtxStartRange("prg_canon_response_p1_dpdmu",3) +#endif call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& &norbsCore,beta,mysyprt(ipt)%estr%evects,& &mysyprt(ipt)%estr%evals,ef,12,norbs) - +#ifdef USE_NVTX + call nvtxEndRange +#endif +#ifdef USE_NVTX + call nvtxStartRange("BML_math_2",4) +#endif !At this point ptham is not needed anymore - if(bml_allocated(ptham_bml)) call bml_deallocate(ptham_bml) + !if(bml_allocated(ptham_bml)) call bml_deallocate(ptham_bml) !Transform P1 back to the nonortho-canonical basis set. call bml_multiply(zq_bml,p1_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) @@ -1018,7 +1052,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !At this point ZQ and ZQt are not needed anymore call bml_deallocate(zq_bml) call bml_deallocate(zqt_bml) - +#ifdef USE_NVTX + call nvtxEndRange +#endif !Here we compute the charges response (q1) from P1 and we store it on !a vector q1 that stores all the previous q1s from past iranks iterations !We also compute the partial trace contribution (trP1) from this mpi @@ -1169,6 +1205,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re ! write(*,*) DOT_PRODUCT(dr_save(:,myi),dr_save(:,myj)) ! enddo ! enddo +#ifdef USE_NVTX + call nvtxStartRange("Deallocations",5) +#endif !Deallocations @@ -1205,7 +1244,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re if(allocated(my_coul_forces_r))deallocate(my_coul_forces_r) if(allocated(K0Res))deallocate(K0Res) if(allocated(K0ResPart))deallocate(K0ResPart) - +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_msI("gpmdcov_get_kernel_byParts","Time for get_kernel_byParts & &"//to_string(mls() - mls_v)//" ms",lt%verbose,myRank) diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 index e6d01850..f0fe0ea5 100644 --- a/examples/gpmdk/src/gpmdcov_mdloop.F90 +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -35,7 +35,20 @@ subroutine gpmdcov_MDloop() real(dp) :: ke_tensor(3,3) real(dp) :: pressure_tensor(3,3) integer :: total_steps + integer :: cuda_error + interface + integer(c_int) function cudaProfilerStart() bind(c,name="cudaProfilerStart") + use iso_c_binding + end function cudaProfilerStart + end interface + + interface + integer(c_int) function cudaProfilerStop() bind(c,name="cudaProfilerStop") + use iso_c_binding + end function cudaProfilerStop + end interface + ! Prepare for Langevin dynamics if needed if(gpmdt%langevin)then @@ -74,7 +87,13 @@ subroutine gpmdcov_MDloop() mls_md = mls() #ifdef USE_NVTX - call nvtxStartRange("MD iter",1) + if (mdstep == 10) then + cuda_error = cudaProfilerStart() + endif + if (mdstep == 20) then + cuda_error = cudaProfilerStop() + endif + call nvtxStartRange("MD_iter",1) #endif mls_md1 = mls() @@ -128,10 +147,11 @@ subroutine gpmdcov_MDloop() write(*,*)"Pressure [bar] = ",pressure_tensor(1,1)+pressure_tensor(2,2)+pressure_tensor(3,3) endif - call gpmdcov_msI("gpmdcov_MDloop","Time for Preliminars "//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_MDloop","Time for Preliminaries "//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) mls_md1 = mls() if(.not.gpmdt%langevin)then + !> First 1/2 of Leapfrog step call gpmdcov_msMem("gpmdcov_mdloop", "Before halfVerlet",lt%verbose,myRank) if(myRank == 1 .and. lt%verbose >= 1) call prg_timer_start(dyn_timer,"Half Verlet") @@ -248,7 +268,13 @@ subroutine gpmdcov_MDloop() & mod(mdstep,kernel%updateEach) == 0)then mls_md2 = mls() call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_rankN_update_byParts",lt%verbose,myRank) +#ifdef USE_NVTX + call nvtxStartRange("RankN_update",2) +#endif call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_rankN_update_byParts",lt%verbose,myRank) call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_rankN_update_byParts "& &//to_string(mls() - mls_md2)//" ms",lt%verbose,myRank) @@ -300,9 +326,21 @@ subroutine gpmdcov_MDloop() call gpmdcov_destroy_nlist(nl,lt%verbose) !call destroy_nlist(nl) if(nlistSparse)then - call gpmdcov_build_nlist_sparse_v2(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) +#ifdef USE_NVTX + call nvtxStartRange("build_nlist_sparse_v2",3) +#endif + call gpmdcov_build_nlist_sparse_v2(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) +#ifdef USE_NVTX + call nvtxEndRange +#endif else - call gpmdcov_build_nlist_full(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) +#ifdef USE_NVTX + call nvtxStartRange("build_nlist_full",3) +#endif + call gpmdcov_build_nlist_full(sy%coordinate,sy%lattice_vector,coulcut,nl,lt%verbose,myRank,numRanks) +#ifdef USE_NVTX + call nvtxEndRange +#endif endif !LBox(1) = sy%lattice_vector(1,1) !LBox(2) = sy%lattice_vector(2,2) @@ -321,14 +359,27 @@ subroutine gpmdcov_MDloop() ! This builds the new graph. mls_md1 = mls() call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_Part",lt%verbose,myRank) - call gpmdcov_Part(2) +#ifdef USE_NVTX + call nvtxStartRange("Part",4) +#endif + + call gpmdcov_Part(2) +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_Part",lt%verbose,myRank) call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_Part & &"//to_string(mls() - mls_i)//" ms",lt%verbose,myRank) !> Reprg_initialize parts. mls_i = mls() call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_InitParts",lt%verbose,myRank) +#ifdef USE_NVTX + call nvtxStartRange("InitParts",5) +#endif call gpmdcov_InitParts() +#ifdef USE_NVTX + call nvtxEndRange +#endif call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_InitParts",lt%verbose,myRank) call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_InitParts & &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) @@ -367,9 +418,12 @@ subroutine gpmdcov_MDloop() sy%net_charge = n +#ifdef USE_NVTX + call nvtxStartRange("DM_min",6) +#endif if(eig)then call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min",lt%verbose,myRank) - call gpmdcov_DM_Min(1,sy%net_charge,.false.) + call gpmdcov_DM_Min(1,sy%net_charge,.false.) call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min",lt%verbose,myRank) else call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min_Eig",lt%verbose,myRank) @@ -380,16 +434,31 @@ subroutine gpmdcov_MDloop() call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_DM_Min_1 & &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) +#ifdef USE_NVTX + call nvtxEndRange +#endif if(kernel%xlbolevel1)then allocate(n1(sy%nats)) if(mdstep > 1)then !sy%net_charge = n - !Compute KK0Res - call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) + !Compute KK0Res +#ifdef USE_NVTX + call nvtxStartRange("RankN_update",2) +#endif + call gpmdcov_rankN_update_byParts(sy%net_charge,n,syprt,syprtk,kernel%rankNUpdate,KK0Res) +#ifdef USE_NVTX + call nvtxEndRange +#endif !Use KK0Res to update n to and n1 n1 = n - KK0Res sy%net_charge = n1 +#ifdef USE_NVTX + call nvtxStartRange("DM_min_eig",6) +#endif call gpmdcov_DM_Min_Eig(1,sy%net_charge,.false.,.false.) +#ifdef USE_NVTX + call nvtxEndRange +#endif !Compute H again and get q_min from n1 later in the code resnorm = norm2(sy%net_charge - n1)/sqrt(dble(sy%nats)) endif @@ -408,7 +477,9 @@ subroutine gpmdcov_MDloop() endif !write(*,*)"Step, Energy, EGap, Resnorm", mdstep, Energy, egap_glob, resnorm endif - +#ifdef USE_NVTX + call nvtxStartRange("EnergAndForces",7) +#endif call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_EnergAndForces",lt%verbose,myRank) if(kernel%xlbolevel1)then if(mdstep <= 1) n1 = n @@ -420,7 +491,9 @@ subroutine gpmdcov_MDloop() call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_EnergAndForces",lt%verbose,myRank) call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_EnergAndForces & &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) - +#ifdef USE_NVTX + call nvtxEndRange +#endif mls_md1 = mls() !> Adjust forces for the linearized XLBOMD functional call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_fcoulupdate",lt%verbose,myRank) diff --git a/examples/gpmdk/src/gpmdcov_neighbor.F90 b/examples/gpmdk/src/gpmdcov_neighbor.F90 index 0f69701f..b48faaee 100644 --- a/examples/gpmdk/src/gpmdcov_neighbor.F90 +++ b/examples/gpmdk/src/gpmdcov_neighbor.F90 @@ -198,6 +198,8 @@ subroutine gpmdcov_build_nlist_full(coords,lattice_vectors,rcut,nl,verbose,rank, vectNrnnlist(i) = cnt enddo !$omp end parallel do + + call prg_barrierParallel() !We do a sum reduction on all the vectors #ifdef DO_MPI From e3f9a44c05b8d0df65ba49f730762d4276bcbee2 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 14:33:49 -0600 Subject: [PATCH 15/69] Debug resizing and resize two more arrays --- examples/gpmdk/src/gpmdcov_kernel.F90 | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 4adf65be..d54bad85 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -776,6 +776,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re implicit none integer :: myi,myj,myjj,irank,mRanks integer :: norbsCore,norbs + integer, save :: maxorbs = -1 integer, intent(in) :: maxRanks integer :: natsCore,natsCoreHalo integer, allocatable :: maxCoresAmongParts(:) @@ -906,7 +907,10 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re c_i = 0.0_dp allocate(ff(maxCoresAmongPartsAndRanks,partsInEachRank(myRank),mRanks)) ff = 0.0_dp - + + if (maxorbs == -1) then + maxorbs = min(maxCoresAmongPartsAndRanks*20,sy%nats)*4 + endif !Here we enter the loop for the rank updates (do not confuse with MPI rank) do irank = 1, mRanks @@ -984,12 +988,13 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call nvtxStartRange("Reallocation event",7) #endif if (.not.bml_allocated(ptham_bml))then - call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,8000) - else - write(*,*)"ptham_bml already allocated, using bml_set_N" - call gpmdcov_bml_set_N(ptham_bml,norbs) - endif - + call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,maxorbs) + call gpmdcov_reallocate_denseBmlRealMat(zq_bml,maxorbs) + call gpmdcov_reallocate_denseBmlRealMat(zqt_bml,maxorbs) + endif + call gpmdcov_bml_set_N(ptham_bml,norbs) + call gpmdcov_bml_set_N(zq_bml,norbs) + call gpmdcov_bml_set_N(zqt_bml,norbs) ! call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) #ifdef USE_NVTX call nvtxEndRange @@ -1002,9 +1007,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !Compute transformations ZQ and (ZQ)^t transformation that takes from the canonical nonorthogonal !to the orthogonal eigenbasis. - call gpmdcov_reallocate_denseBmlRealMat(zq_bml,norbs) call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,1.0_dp,0.0_dp,lt%threshold) - call gpmdcov_reallocate_denseBmlRealMat(zqt_bml,norbs) call bml_transpose(zq_bml,zqt_bml) !Take H1 to the ortho-eigen basis set. @@ -1050,8 +1053,8 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call bml_multiply(ptaux_bml,zqt_bml,dPdMu_bml,2.0_dp,0.0_dp,0.0_dp) !At this point ZQ and ZQt are not needed anymore - call bml_deallocate(zq_bml) - call bml_deallocate(zqt_bml) + !call bml_deallocate(zq_bml) + !call bml_deallocate(zqt_bml) #ifdef USE_NVTX call nvtxEndRange #endif From 2c63bedb942424f050a3c4c2aca824ae67551aa2 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 15:43:03 -0600 Subject: [PATCH 16/69] Resize zqt array and use new bml_transpose_inplace to avoid allocation --- examples/gpmdk/src/gpmdcov_kernel.F90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index d54bad85..0abd23dd 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -1008,7 +1008,8 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !Compute transformations ZQ and (ZQ)^t transformation that takes from the canonical nonorthogonal !to the orthogonal eigenbasis. call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,1.0_dp,0.0_dp,lt%threshold) - call bml_transpose(zq_bml,zqt_bml) + call bml_copy(zq_bml,zqt_bml) + call bml_transpose_inplace(zqt_bml) !Take H1 to the ortho-eigen basis set. call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) @@ -1054,7 +1055,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !At this point ZQ and ZQt are not needed anymore !call bml_deallocate(zq_bml) - !call bml_deallocate(zqt_bml) + !if(bml_allocated(zqt_bml)) call bml_deallocate(zqt_bml) #ifdef USE_NVTX call nvtxEndRange #endif From a04428298647b9a27a558ae376f3d92d3289297d Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 16:18:58 -0600 Subject: [PATCH 17/69] Move response into gpmdk in preparation for GPU kernel --- examples/gpmdk/src/CMakeLists.txt | 1 + examples/gpmdk/src/gpmdcov_kernel.F90 | 16 ++- examples/gpmdk/src/gpmdcov_response.F90 | 127 ++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 examples/gpmdk/src/gpmdcov_response.F90 diff --git a/examples/gpmdk/src/CMakeLists.txt b/examples/gpmdk/src/CMakeLists.txt index f9293dbc..7356e59c 100644 --- a/examples/gpmdk/src/CMakeLists.txt +++ b/examples/gpmdk/src/CMakeLists.txt @@ -165,6 +165,7 @@ list( "gpmdcov_highorder.F90" "gpmdcov_field.F90" "gpmdcov_nvtx.F90" +"gpmdcov_response.F90" ) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 0abd23dd..ca3a328c 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -10,6 +10,7 @@ module gpmdcov_kernel_mod use prg_kernelparser_mod use gpmdcov_allocation_mod use gpmdcov_writeout_mod + use gpmdcov_response_mod #ifdef USE_NVTX use gpmdcov_nvtx_mod @@ -645,7 +646,10 @@ subroutine gpmdcov_getKernel_byParts(mysyprt,mysyprtk) !call prg_canon_response2_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& ! &mysyprt(ipt)%estr%evals,ef,12,norbs,lt%threshold) - call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + !call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& + ! &mysyprt(ipt)%estr%evals,ef,12,norbs) + call gpmdcov_response_dpdmu(p1_bml,dPdMu,ptham_bml,& &norbsCore,beta,mysyprt(ipt)%estr%evects,& &mysyprt(ipt)%estr%evals,ef,12,norbs) call bml_get_diagonal(p1_bml,p1_dia) @@ -991,10 +995,12 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,maxorbs) call gpmdcov_reallocate_denseBmlRealMat(zq_bml,maxorbs) call gpmdcov_reallocate_denseBmlRealMat(zqt_bml,maxorbs) + call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) endif call gpmdcov_bml_set_N(ptham_bml,norbs) call gpmdcov_bml_set_N(zq_bml,norbs) call gpmdcov_bml_set_N(zqt_bml,norbs) + call gpmdcov_bml_set_N(dPdMu_bml,norbs) ! call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) #ifdef USE_NVTX call nvtxEndRange @@ -1027,9 +1033,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re ! &norbsCore,beta,mysyprt(ipt)%estr%evects,& ! &mysyprt(ipt)%estr%evals,ef,12,norbs,lt%threshold) #ifdef USE_NVTX - call nvtxStartRange("prg_canon_response_p1_dpdmu",3) + call nvtxStartRange("gpmdcov_response_dpdmu",3) #endif - call prg_canon_response_p1_dpdmu(p1_bml,dPdMu,ptham_bml,& + call gpmdcov_response_dpdmu(p1_bml,dPdMu,ptham_bml,& &norbsCore,beta,mysyprt(ipt)%estr%evects,& &mysyprt(ipt)%estr%evals,ef,12,norbs) #ifdef USE_NVTX @@ -1046,7 +1052,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call bml_multiply(ptaux_bml,zqt_bml,p1_bml,2.0_dp,0.0_dp,0.0_dp) !Since dPdMu is represented by a vector, we will convert it into a BML matrix - call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) + !call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) call bml_set_diagonal(dPdMu_bml,dPdMu) !Transform dPdMu back to the nonortho-canonical basis set @@ -1083,7 +1089,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re mysyprt(ipt)%spindex, norbs, lt%threshold) dqdmu(:,iptt) = ptnet_charge(1:natsCore) trdPdMu = trdPdMu + sum(ptnet_charge(1:natsCore)) - call bml_deallocate(dPdMu_bml) + !call bml_deallocate(dPdMu_bml) call bml_deallocate(ptaux_bml) deallocate(ptnet_charge) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 new file mode 100644 index 00000000..a06b6e33 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -0,0 +1,127 @@ +!> Module to compute the density matrix response and related quantities. +!! \ingroup GPMDK +!! \todo Add the response scf +!! \todo Change name response_SP2 to dm_prt_response +!! \todo Change name response_rs to rs_prt_response +!! \brief More information about the theory can be found at \cite Niklasson2005 and Niklasson2015 +module gpmdcov_response_mod + + use bml + use prg_kernelparser_mod + use prg_densitymatrix_mod + use prg_openfiles_mod + + implicit none + + private + + integer, parameter :: dp = kind(1.0d0) + real(dp), parameter :: pi = 3.14159265358979323846264338327950_dp + + type, public :: RespData_type + character(20) :: RespMode + character(20) :: TypeOfPert + character(20) :: BmlType + integer :: Mdim + real(dp) :: NumThresh + logical :: ComputeDipole + logical :: GetResponse + real(dp) :: FieldIntensity + real(dp) :: field(3) + end type RespData_type + + public :: gpmdcov_response_dpdmu + +contains + + !> First-order Canonical Density Matrix Perturbation Theory. + !! \brief Assuming known mu0 and representation in H0's eigenbasis Q, + !! where H0 and P0 become diagonal. + !! (mu0, eigenvalues e and eigenvectors Q of H0 are assumed known) + !! Based on PRL 92, 193001 (2004) and PRE 92, 063301 (2015) + subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0,m,HDIM) + + type(bml_matrix_t), intent(inout) :: P1_bml, H1_bml + type(bml_matrix_t) :: dx1_bml + type(bml_matrix_t) :: Q_bml + real(dp), parameter :: ONE = 1.D0, TWO = 2.D0, ZERO = 0.D0 + integer, intent(in) :: HDIM, m, Norbs ! = Number of occupied orbitals,m= Number of recursion steps + real(dp), intent(in) :: evals(HDIM) !Q and e are eigenvectors and eigenvalues of H0 + real(dp), intent(inout) :: dPdMu(HDIM) !Q and e are eigenvectors and eigenvalues of H0 + ! real(PREC) :: P1(HDIM,HDIM) ! Density matrix response + ! derivative with respect to perturbation H1 to H0 + real(dp), intent(in) :: beta, mu0 ! Electronic temperature and chemicalpotential + real(dp), allocatable :: P1(:,:), DX1(:,:) + real(dp) :: h_0(HDIM), p_0(HDIM),p_02(HDIM),iD0(HDIM) + real(dp) :: cnst, kB, mu1, sumdPdmu + integer :: i, j, k + character(20) :: bml_type + + kB = 8.61739e-5 ! (eV/K) + h_0 = evals ! Diagonal Hamiltonian H0 respresented in the eigenbasis Q + cnst = beta/(1.D0*2**(m+2)) ! Scaling constant + p_0 = 0.5D0 + cnst*(h_0-mu0) ! Initialization for P0 represented in eigenbasis Q + call bml_copy_new(H1_bml,P1_bml) + call bml_copy_new(H1_bml,DX1_bml) + call bml_scale(-cnst,P1_bml) !(set mu1 = 0 for simplicity) !Initialization of DM response in Q representation (not diagonal in Q) + + allocate(P1(HDIM,HDIM)) + allocate(DX1(HDIM,HDIM)) + + P1 = 0.0_dp + DX1 = 0.0_dp + + call bml_export_to_dense(P1_bml,P1) + call bml_export_to_dense(DX1_bml,DX1) + +!!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) + + do i = 1,m ! Loop over m recursion steps + !p_02 = p_0*p_0 + !$omp parallel do default(none) private(k) & + !$omp private(j) & + !$omp firstprivate(p_0) & + !$omp shared(HDIM,P1,DX1) + !!$acc parallel loop deviceptr(P1,DX1,p_0) + !!$acc parallel loop + do k = 1,HDIM + DX1(:,k) = (p_0(:) + p_0(k))*P1(:,k) + !DX1(k,:) = (p_0(k) + p_0(:))*P1(k,:) + enddo + !!$acc end parallel loop + !$omp end parallel do + !iD0 = 1.D0/(2.D0*(p_0*p_0-p_0)+1.D0) + + !$omp parallel do default(none) private(k) & + !$omp private(j) & + !$omp firstprivate(iD0,p_0) & + !$omp shared(HDIM,P1,DX1) + !!$acc parallel loop deviceptr(P1,DX1,p_0) + !!$acc parallel loop + do k = 1,HDIM + P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) + !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) + enddo + !!$acc end parallel loop + !$omp end parallel do + !!$acc kernels deviceptr(p_0) + !!$acc kernels + p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) + !!$acc end kernels + enddo + +!!$acc end data + + bml_type = bml_get_type(P1_bml) + call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml + + deallocate(P1) + deallocate(DX1) + + call bml_deallocate(DX1_bml) + + dPdmu = beta*p_0*(1.D0-p_0) + + end subroutine gpmdcov_response_dpdmu + +end module gpmdcov_response_mod From dfeb3a17e43ea2fd0d8d2b99e01e774976b23c27 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 16:27:48 -0600 Subject: [PATCH 18/69] Add some useful build scripts --- build_progress_nvhpc.sh | 45 +++++++++++++++++++ examples/gpmdk/build_gpmd_nvhpc.sh | 9 ++++ examples/gpmdk/build_venado_hackathon_nvtx.sh | 13 ++++++ scripts/build_magma.sh | 4 ++ scripts/build_magma_nvhpc.sh | 8 ++++ 5 files changed, 79 insertions(+) create mode 100755 build_progress_nvhpc.sh create mode 100755 examples/gpmdk/build_gpmd_nvhpc.sh create mode 100755 examples/gpmdk/build_venado_hackathon_nvtx.sh create mode 100644 scripts/build_magma.sh create mode 100644 scripts/build_magma_nvhpc.sh diff --git a/build_progress_nvhpc.sh b/build_progress_nvhpc.sh new file mode 100755 index 00000000..c6786d60 --- /dev/null +++ b/build_progress_nvhpc.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Make a copy of this script and modify the METIS and BML library locations + +rm -r build +rm -r install_nvhpc + +# Set METIS and BML Library locations +METIS_LIB="/usr/projects/icapt/mewall/venado/packages/metis-5.1.0/install" +BML_LIB="/usr/projects/icapt/mewall/venado/packages/qmd-progress/bml/install_nvhpc" +#MPI_LIB="/projects/darwin-nv/rhel9/aarch64/packages/nvhpc/Linux_aarch64/24.3/comm_libs/openmpi/openmpi-3.1.5/lib" + +MY_PATH=`pwd` + +# Configuring PROGRESS with OpenMP +export CC=$(which mpicc) +export FC=$(which mpifort) +export CXX=$(which mpic++) +#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ffixed-line-length-none"} +#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"} +export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$BML_LIB/lib/pkgconfig:$BML_LIB/lib64/pkgconfig" +export PROGRESS_OPENMP=${PROGRESS_OPENMP:=yes} +export PROGRESS_MPI=${PROGRESS_MPI:=yes} +export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_nvhpc"} +export PROGRESS_GRAPHLIB=${PROGRESS_GRAPHLIB:=yes} +export PROGRESS_TESTING=${PROGRESS_TESTING:=yes} +export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release} +export PROGRESS_EXAMPLES=${PROGRESS_EXAMPLES:=yes} +#export PROGRESS_BENCHMARKS=yes +export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=$BML_LIB/:$METIS_LIB/} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch -fdefault-integer-8"} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-I${MPI_LIB} -g -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-L${MPI_LIB} -lmpi -lmpi_mpifh -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +#export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-I${MPI_LIB} -g"} +#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-L${MPI_LIB} -lmpi -lmpi_mpifh"} +export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:=""} +export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:=""} +export CMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH:="$METIS_LIB/include"} +export CMAKE_LIBRARY_PATH=${CMAKE_LIBRARY_PATH:="$METIS_LIB/lib"} +#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" +./build.sh configure + +# Make PROGRESS library and examples after running this script: +# cd build +# make diff --git a/examples/gpmdk/build_gpmd_nvhpc.sh b/examples/gpmdk/build_gpmd_nvhpc.sh new file mode 100755 index 00000000..cd5f5779 --- /dev/null +++ b/examples/gpmdk/build_gpmd_nvhpc.sh @@ -0,0 +1,9 @@ +WORKDIR=/usr/projects/icapt/mewall/venado/packages +rm -rf build_nvhpc +mkdir build_nvhpc +cd build_nvhpc +cmake -DCMAKE_Fortran_COMPILER="mpifort" -DPROGRESS_MPI="yes" -DGPMDK_NVTX="yes" \ +-DEXTRA_FCFLAGS="-g -mp -L${NVHPC_ROOT}/cuda/lib64 -lnvToolsExt" \ +-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_nvhpc/;$WORKDIR/qmd-progress/bml/install_nvhpc;$WORKDIR/metis-5.1.0/install" ../src/ +make + diff --git a/examples/gpmdk/build_venado_hackathon_nvtx.sh b/examples/gpmdk/build_venado_hackathon_nvtx.sh new file mode 100755 index 00000000..fc9c6cbe --- /dev/null +++ b/examples/gpmdk/build_venado_hackathon_nvtx.sh @@ -0,0 +1,13 @@ +WORKDIR=/usr/projects/icapt/mewall/venado/packages +rm -rf build_hackathon +mkdir build_hackathon +cd build_hackathon +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ +-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ +-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ +#-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ +make -j1 + +#cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ +#-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ +#-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ diff --git a/scripts/build_magma.sh b/scripts/build_magma.sh new file mode 100644 index 00000000..d22dd566 --- /dev/null +++ b/scripts/build_magma.sh @@ -0,0 +1,4 @@ +rm -rf build +mkdir build +cd build +CMAKE_PREFIX_PATH=$CUDA_HOME/../../math_libs/$CRAY_CUDATOOLKIT_VERSION cmake .. -DCMAKE_INSTALL_PREFIX=/usr/projects/icapt/mewall/venado/packages/magma-2.7.2/install -DGPU_TARGET=sm_90 diff --git a/scripts/build_magma_nvhpc.sh b/scripts/build_magma_nvhpc.sh new file mode 100644 index 00000000..ec31a652 --- /dev/null +++ b/scripts/build_magma_nvhpc.sh @@ -0,0 +1,8 @@ +WORKDIR=/usr/projects/icapt/mewall/venado/packages +rm -rf build_nvhpc +mkdir build_nvhpc +cd build_nvhpc +#cmake .. -DCMAKE_INSTALL_PREFIX=$WORKDIR/magma-2.7.2/install_nvhpc -DGPU_TARGET=sm_90 -DCMAKE_CUDA_ARCHITECTURES=90 -DLAPACK_LIBRARIES="-L/projects/darwin-nv/rhel9/aarch64/packages/nvpl/23.11/lib -lnvpl_lapack_lp64_gomp -lnvpl_blas_lp64_gomp" +CMAKE_PREFIX_PATH=$NVHPC_ROOT/math_libs/lib64 cmake .. -DCMAKE_INSTALL_PREFIX=$WORKDIR/magma-2.7.2/install_nvhpc -DGPU_TARGET=sm_90 -DCMAKE_CUDA_ARCHITECTURES=90 +make -j16 install +cd - From 1aebd5913a3e0f6ea2b32a536b4f63cf496cca4e Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 16:30:49 -0600 Subject: [PATCH 19/69] Update bml submodule --- bml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bml b/bml index 25bf4f6f..7aaf972b 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 25bf4f6fbfdcd2ff4266c233c7dd1d9e77b4be1f +Subproject commit 7aaf972b755ad28dbc592b8a6e17b26237f68bc2 From 15594cf277c66836bbd42a1b9ad934441d13431e Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 16:33:35 -0600 Subject: [PATCH 20/69] Reduce allocations in hcsf method --- src/prg_charges_mod.F90 | 68 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/prg_charges_mod.F90 b/src/prg_charges_mod.F90 index 3c2efb98..a5cac108 100644 --- a/src/prg_charges_mod.F90 +++ b/src/prg_charges_mod.F90 @@ -16,7 +16,7 @@ module prg_charges_mod integer, parameter :: dp = kind(1.0d0) - public :: prg_get_charges, prg_get_hscf, prg_get_hscf_v2 + public :: prg_get_charges, prg_get_hscf, prg_get_hscf_v2, prg_get_hscf_maxorb contains @@ -205,5 +205,71 @@ subroutine prg_get_hscf_v2(ham0_bml,over_bml,ham_bml,spindex,hindex,hubbardu,cha end subroutine prg_get_hscf_v2 + subroutine prg_get_hscf_maxorb(ham0_bml,over_bml,ham_bml,spindex,hindex,hubbardu,charges,& + coulomb_pot_r,coulomb_pot_k,mdimin,threshold,maxorb) + character(20) :: bml_type + integer :: i, j, nats, norb, mdim + integer, intent(in) :: hindex(:,:), mdimin, spindex(:), maxorb + real(dp), allocatable :: coulomb_pot(:), diagonal(:) + real(dp), intent(in) :: charges(:), coulomb_pot_k(:),coulomb_pot_r(:), hubbardu(:) + real(dp), intent(in) :: threshold + real(dp), allocatable, save :: zero(:,:) + type(bml_matrix_t), save :: zero_bml, aux_bml + type(bml_matrix_t), intent(in) :: ham0_bml, over_bml + type(bml_matrix_t), intent(inout) :: ham_bml + + nats = size(coulomb_pot_r) + allocate(coulomb_pot(nats)) + + bml_type = bml_get_type(ham_bml) + + if(.not.bml_allocated(zero_bml))then + allocate(zero(maxorb,maxorb)) + zero = 0.0_dp + call bml_import_from_dense(bml_type,zero,zero_bml) + call bml_import_from_dense(bml_type,zero,aux_bml) + endif + + norb = bml_get_N(ham0_bml) + if(hindex(2,nats) .ne. norb)then + write(*,*)"ERROR in prg_get_hscf. hindex incompatible with ham0" + stop + endif + + if(mdimin.ne.norb)then + write(*,*)"ERROR in prg_ge_hscf_maxorb: mdim must equal norb" + stop + endif + + call bml_set_N_dense(aux_bml,norb) + call bml_set_N_dense(zero_bml,norb) + call bml_copy(zero_bml,aux_bml) + + allocate(diagonal(norb)) + + call bml_copy(ham0_bml,ham_bml) + + bml_type = bml_get_type(ham_bml) + + coulomb_pot = coulomb_pot_k + coulomb_pot_r + + + do i = 1,nats + do j = hindex(1,i),hindex(2,i) + diagonal(j) = hubbardu(spindex(i))*charges(i) + coulomb_pot(i) + enddo + enddo + + call bml_set_diagonal(aux_bml,diagonal) + call bml_multiply(over_bml,aux_bml,ham_bml,0.5_dp,1.0_dp,threshold) ! h = h+ 0.5*s*h1 + + call bml_multiply(aux_bml,over_bml,ham_bml,0.5_dp,1.0_dp,threshold) ! h = h+ 0.5*h1*s + + deallocate(diagonal) + deallocate(coulomb_pot) + !call bml_deallocate(aux_bml) + + end subroutine prg_get_hscf_maxorb + end module prg_charges_mod From 11695ecf51ce0d1b7e47625a8cc65f8d0120421d Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 19 Jul 2024 16:45:19 -0600 Subject: [PATCH 21/69] Update build script --- examples/gpmdk/build_venado_hackathon_nvtx.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/gpmdk/build_venado_hackathon_nvtx.sh b/examples/gpmdk/build_venado_hackathon_nvtx.sh index fc9c6cbe..72391d0f 100755 --- a/examples/gpmdk/build_venado_hackathon_nvtx.sh +++ b/examples/gpmdk/build_venado_hackathon_nvtx.sh @@ -3,11 +3,8 @@ rm -rf build_hackathon mkdir build_hackathon cd build_hackathon cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ --DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ +-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ #-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ make -j1 -#cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ -#-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ -#-DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ From f1d152c7a510caf0e5615811ca08ed9eb3ee89c6 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 22 Jul 2024 09:28:41 -0600 Subject: [PATCH 22/69] Modifications to support the new bml_transpose Fortran API --- bml | 2 +- examples/gploop/gploop.F90 | 2 +- examples/gpmd/gpmd.F90 | 2 +- examples/gpmdcov/gpmdcov.F90 | 2 +- examples/gpmdk/run/TrpCage/input.in | 6 +++--- examples/gpmdk/run/water/input.in | 2 +- examples/gpmdk/src/gpmdcov_kernel.F90 | 8 ++++---- examples/gpscfcov/gpscfcov.F90 | 2 +- examples/mdresponse/mdresponse.F90 | 2 +- src/prg_densitymatrix_mod.F90 | 20 ++++++++++---------- src/prg_genz_mod.F90 | 2 +- src/prg_implicit_fermi_mod.F90 | 4 ++-- src/prg_modelham_mod.F90 | 4 ++-- src/prg_nonortho_mod.F90 | 4 ++-- src/prg_pulaycomponent_mod.F90 | 2 +- src/prg_response_mod.F90 | 6 +++--- src/prg_xlbokernel_mod.F90 | 26 +++++++++++++------------- 17 files changed, 48 insertions(+), 48 deletions(-) diff --git a/bml b/bml index 7aaf972b..43cc151d 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 7aaf972b755ad28dbc592b8a6e17b26237f68bc2 +Subproject commit 43cc151d873069e299314a5e08d117c8ec6f1169 diff --git a/examples/gploop/gploop.F90 b/examples/gploop/gploop.F90 index f5c78c5a..4cf9b6f7 100644 --- a/examples/gploop/gploop.F90 +++ b/examples/gploop/gploop.F90 @@ -268,7 +268,7 @@ program gploop !> Symmetrize and Threshold the Matrix call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,copy_g_bml) call bml_threshold(g_bml, gsp2%gthreshold) - call bml_transpose(g_bml, copy_g_bml) + call bml_transpose_new(g_bml, copy_g_bml) call bml_add_deprecated(0.5_dp,g_bml,0.5_dp,copy_g_bml,0.0_dp) call bml_threshold(g_bml, gsp2%gthreshold) call bml_deallocate(copy_g_bml) diff --git a/examples/gpmd/gpmd.F90 b/examples/gpmd/gpmd.F90 index fd10633b..23ea00b3 100644 --- a/examples/gpmd/gpmd.F90 +++ b/examples/gpmd/gpmd.F90 @@ -1089,7 +1089,7 @@ subroutine gpmd_graphpart !> Symmetrize and Threshold the Matrix call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,copy_g_bml) call bml_threshold(g_bml, gsp2%gthreshold) - call bml_transpose(g_bml, copy_g_bml) + call bml_transpose_new(g_bml, copy_g_bml) call bml_add_deprecated(0.5_dp,g_bml,0.5_dp,copy_g_bml,0.0_dp) call bml_threshold(g_bml, gsp2%gthreshold) call bml_deallocate(copy_g_bml) diff --git a/examples/gpmdcov/gpmdcov.F90 b/examples/gpmdcov/gpmdcov.F90 index 18c0a0d7..31a1d84d 100644 --- a/examples/gpmdcov/gpmdcov.F90 +++ b/examples/gpmdcov/gpmdcov.F90 @@ -1295,7 +1295,7 @@ subroutine gpmd_graphpart ! call bml_write_matrix(g_bml,"g_bml_bef") ! call bml_zero_matrix(gsp2%bml_type,bml_element_real,kind(1.0),sy%nats,mdim,copy_g_bml) ! call bml_threshold(g_bml, gsp2%gthreshold) - ! call bml_transpose(g_bml, copy_g_bml) + ! call bml_transpose_new(g_bml, copy_g_bml) ! call bml_add_deprecated(0.5_dp,g_bml,0.5_dp,copy_g_bml,0.0_dp) ! call bml_threshold(g_bml, gsp2%gthreshold) ! call bml_deallocate(copy_g_bml) diff --git a/examples/gpmdk/run/TrpCage/input.in b/examples/gpmdk/run/TrpCage/input.in index b05d89c7..22136780 100644 --- a/examples/gpmdk/run/TrpCage/input.in +++ b/examples/gpmdk/run/TrpCage/input.in @@ -120,7 +120,7 @@ GSP2{ NLGraphCut= 4.5 CovGraphFact= 4.5 #CovGraphFact= 3.0 - PartitionCount= 256 + PartitionCount= 64 #PartitionCount= 8 #PartitionCount= 512 #PartitionCount= 16 @@ -202,7 +202,7 @@ GPMD{ UseSMD= F HtoD= T UseVectSKBlock= F - ReplicateX= 2 - ReplicateY= 2 + ReplicateX= 1 + ReplicateY= 1 ReplicateZ= 1 } diff --git a/examples/gpmdk/run/water/input.in b/examples/gpmdk/run/water/input.in index 7d1f61b5..dc41c396 100644 --- a/examples/gpmdk/run/water/input.in +++ b/examples/gpmdk/run/water/input.in @@ -176,7 +176,7 @@ GPMD{ WriteTrajectory= T WriteCoordsEach= 10 LangevinMethod= Siva - LangevinDynamics= T + LangevinDynamics= F LangevinGamma= 0.01 InitialTemperature= 300.0 } diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index ca3a328c..aefad0ca 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -219,7 +219,7 @@ subroutine gpmdcov_getKernel(nats) deallocate(ptcoul_pot_r) call bml_multiply(syprt(ipt)%estr%zmat,syprt(ipt)%estr%evects,zq_bml, 1.0_dp,0.0_dp,lt%threshold) - call bml_transpose(zq_bml,zqt_bml) + call bml_transpose_new(zq_bml,zqt_bml) call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) @@ -409,7 +409,7 @@ subroutine gpmdcov_getKernel_byBlocks(nats) mlsi = mls() call bml_multiply(syprt(ipt)%estr%zmat,syprt(ipt)%estr%evects,zq_bml, 1.0_dp,0.0_dp,lt%threshold) - call bml_transpose(zq_bml,zqt_bml) + call bml_transpose_new(zq_bml,zqt_bml) call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) call gpmdcov_msI("gpmdcov_get_kernel_byBlocks","Time for tranf to eigen basis " & @@ -635,7 +635,7 @@ subroutine gpmdcov_getKernel_byParts(mysyprt,mysyprtk) mlsi = mls() call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,& &1.0_dp,0.0_dp,lt%threshold) - call bml_transpose(zq_bml,zqt_bml) + call bml_transpose_new(zq_bml,zqt_bml) call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,lt%threshold) call bml_multiply(ptaux_bml,zq_bml,ptham_bml,1.0_dp,0.0_dp,lt%threshold) call gpmdcov_msIII("gpmdcov_get_kernel_byParts","Time for trasnf to eigenbasis& @@ -1015,7 +1015,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !to the orthogonal eigenbasis. call bml_multiply(mysyprt(ipt)%estr%zmat,mysyprt(ipt)%estr%evects,zq_bml,1.0_dp,0.0_dp,lt%threshold) call bml_copy(zq_bml,zqt_bml) - call bml_transpose_inplace(zqt_bml) + call bml_transpose(zqt_bml) !Take H1 to the ortho-eigen basis set. call bml_multiply(zqt_bml,ptham_bml,ptaux_bml,1.0_dp,0.0_dp,0.0_dp) diff --git a/examples/gpscfcov/gpscfcov.F90 b/examples/gpscfcov/gpscfcov.F90 index 3331476b..45398c1a 100644 --- a/examples/gpscfcov/gpscfcov.F90 +++ b/examples/gpscfcov/gpscfcov.F90 @@ -725,7 +725,7 @@ subroutine gpmd_GraphPart !> Symmetrize and Threshold the Matrix call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,copy_g_bml) call bml_threshold(g_bml, gsp2%gthreshold) - call bml_transpose(g_bml, copy_g_bml) + call bml_transpose_new(g_bml, copy_g_bml) call bml_add_deprecated(0.5_dp,g_bml,0.5_dp,copy_g_bml,0.0_dp) call bml_threshold(g_bml, gsp2%gthreshold) call bml_deallocate(copy_g_bml) diff --git a/examples/mdresponse/mdresponse.F90 b/examples/mdresponse/mdresponse.F90 index 36eb80c3..dd8d943f 100644 --- a/examples/mdresponse/mdresponse.F90 +++ b/examples/mdresponse/mdresponse.F90 @@ -1089,7 +1089,7 @@ subroutine gpmd_graphpart !> Symmetrize and Threshold the Matrix call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,copy_g_bml) call bml_threshold(g_bml, gsp2%gthreshold) - call bml_transpose(g_bml, copy_g_bml) + call bml_transpose_new(g_bml, copy_g_bml) call bml_add_deprecated(0.5_dp,g_bml,0.5_dp,copy_g_bml,0.0_dp) call bml_threshold(g_bml, gsp2%gthreshold) call bml_deallocate(copy_g_bml) diff --git a/src/prg_densitymatrix_mod.F90 b/src/prg_densitymatrix_mod.F90 index 379fc95f..c3197590 100644 --- a/src/prg_densitymatrix_mod.F90 +++ b/src/prg_densitymatrix_mod.F90 @@ -91,7 +91,7 @@ subroutine prg_build_density_T0(ham_bml, rho_bml, threshold, bndfil, eigenvalues call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,aux2_bml,bml_dmode) call bml_multiply(eigenvectors_bml, aux1_bml, aux2_bml, 1.0_dp, 0.0_dp, threshold) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_deallocate(eigenvectors_bml) call bml_multiply(aux2_bml, aux1_bml, rho_bml, 1.0_dp, 0.0_dp, threshold) @@ -173,7 +173,7 @@ subroutine prg_build_density_T(ham_bml, rho_bml, threshold, bndfil, kbt, ef, eig call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,aux2_bml) call bml_multiply(eigenvectors_bml, aux1_bml, aux2_bml, 1.0_dp, 0.0_dp, threshold) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_deallocate(eigenvectors_bml) call bml_multiply(aux2_bml, aux1_bml, rho_bml, 1.0_dp, 0.0_dp, threshold) @@ -258,7 +258,7 @@ subroutine prg_build_density_T_fulldata(ham_bml, rho_bml, threshold, bndfil, kbt call bml_deallocate(occupation_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux1_bml) - call bml_transpose(evects_bml, aux1_bml) + call bml_transpose_new(evects_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, rho_bml, 1.0_dp, 0.0_dp, threshold) call bml_deallocate(aux_bml) @@ -344,7 +344,7 @@ subroutine prg_build_density_T_ed(ham_bml, rho_bml, evects_bml, threshold, bndfi call bml_deallocate(occupation_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux1_bml) - call bml_transpose(evects_bml, aux1_bml) + call bml_transpose_new(evects_bml, aux1_bml) allocate(row(norb)) dvals = 0.0_dp @@ -414,7 +414,7 @@ subroutine prg_get_evalsDvalsEvects(ham_bml, threshold, hindex,& !deallocate(ham) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux1_bml) - call bml_transpose(evects_bml, aux1_bml) + call bml_transpose_new(evects_bml, aux1_bml) allocate(aux(norb,norb)) call bml_export_to_dense(aux1_bml,aux) call bml_deallocate(aux1_bml) @@ -485,7 +485,7 @@ subroutine prg_build_density_fromEvalsAndEvects(evects_bml, evals, rho_bml, thre call bml_deallocate(occupation_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux1_bml) - call bml_transpose(evects_bml, aux1_bml) + call bml_transpose_new(evects_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, rho_bml, 1.0_dp, 0.0_dp, threshold) @@ -551,7 +551,7 @@ subroutine prg_build_density_T_Fermi(ham_bml, rho_bml, threshold, kbt, ef, verbo call bml_deallocate(occupation_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,mdim,aux1_bml) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, rho_bml, 1.0_dp, 0.0_dp, threshold) @@ -566,7 +566,7 @@ subroutine prg_build_density_T_Fermi(ham_bml, rho_bml, threshold, kbt, ef, verbo call bml_multiply(eigenvectors_bml, occupation_bml, aux_bml, 1.0_dp, 0.0_dp, threshold) call bml_zero_matrix(bml_type,bml_element_real,dp,norb,mdim,aux1_bml) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, occupation_bml, 1.0_dp, 0.0_dp, threshold) drho = bml_trace(occupation_bml) @@ -946,7 +946,7 @@ subroutine prg_toEigenspace(mat_bml,matEig_bml,evects_bml,threshold,verbose) & bml_get_distribution_mode(mat_bml)) endif !Do the operations in bml - call bml_transpose(evects_bml, aux_bml) + call bml_transpose_new(evects_bml, aux_bml) call bml_multiply(aux_bml, mat_bml, matEig_bml, 1.0_dp, 0.0_dp,threshold) !U^t*O @@ -993,7 +993,7 @@ subroutine prg_toCanonicalspace(mat_bml,matCan_bml,evects_bml,threshold,verbose) !Do the operations in bml - call bml_transpose(evects_bml, aux_bml) + call bml_transpose_new(evects_bml, aux_bml) call bml_multiply(mat_bml, aux_bml, matCan_bml, 1.0_dp, 0.0_dp,threshold) !O*U^t call bml_multiply(evects_bml,matCan_bml, aux_bml, 1.0_dp, 0.0_dp,threshold) !U*O * U^t diff --git a/src/prg_genz_mod.F90 b/src/prg_genz_mod.F90 index 98602711..984304f1 100644 --- a/src/prg_genz_mod.F90 +++ b/src/prg_genz_mod.F90 @@ -757,7 +757,7 @@ subroutine prg_genz_sp_ref(smat_bml,zmat_bml,nref,norb,bml_type,threshold) do k = 1, NREF !Iterative refinement !Enforcing symmetry (in bml). - call bml_transpose(zmat_bml, xmat_t_bml) !Z^t + call bml_transpose_new(zmat_bml, xmat_t_bml) !Z^t call bml_add_deprecated(0.50_dp,zmat_bml, 0.50_dp, xmat_t_bml,threshold) !(Z^t+Z)/2 call bml_multiply(smat_bml,zmat_bml,temp_bml, 1.0_dp, 0.0_dp,threshold) !S*Z diff --git a/src/prg_implicit_fermi_mod.F90 b/src/prg_implicit_fermi_mod.F90 index a00057da..a0068daf 100644 --- a/src/prg_implicit_fermi_mod.F90 +++ b/src/prg_implicit_fermi_mod.F90 @@ -1086,7 +1086,7 @@ subroutine prg_get_density_matrix(ham_bml, p_bml, beta, mu, threshold) call bml_set_diagonal(occupation_bml, eigenvalues) call bml_multiply(eigenvectors_bml, occupation_bml, aux_bml, 1.0_dp, 0.0_dp,threshold) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, p_bml, 1.0_dp, 0.0_dp, threshold) call bml_deallocate(eigenvectors_bml) @@ -1151,7 +1151,7 @@ subroutine prg_test_density_matrix(ham_bml, p_bml, beta, mu, nocc, osteps, occEr call bml_set_diagonal(occupation_bml, eigenvalues) call bml_multiply(eigenvectors_bml, occupation_bml, aux_bml, 1.0_dp, 0.0_dp,threshold) - call bml_transpose(eigenvectors_bml, aux1_bml) + call bml_transpose_new(eigenvectors_bml, aux1_bml) call bml_multiply(aux_bml, aux1_bml, p_bml, 1.0_dp, 0.0_dp, threshold) call bml_print_matrix("test density",p_bml,0,10,0,10) trdPdmu = bml_trace(p_bml) diff --git a/src/prg_modelham_mod.F90 b/src/prg_modelham_mod.F90 index c85ac656..8c2eadfa 100644 --- a/src/prg_modelham_mod.F90 +++ b/src/prg_modelham_mod.F90 @@ -175,7 +175,7 @@ subroutine prg_twolevel_model(ea, eb, dab, daiaj, dbibj, dec, rcoeff, reshuffle, !Symmetrization call bml_copy_new(h_bml,ht_bml) - call bml_transpose(h_bml,ht_bml) + call bml_transpose_new(h_bml,ht_bml) if(verbose.gt.0)then call bml_print_matrix("h_bml",h_bml,0,10,0,10) call bml_print_matrix("ht_bml",ht_bml,0,10,0,10) @@ -304,7 +304,7 @@ subroutine prg_twolevel_model3d(ea, eb, dab, daiaj, dbibj, dec, rcoeff, reshuffl !Symmetrization (necessary when random factor used) call bml_copy_new(h_bml,ht_bml) - call bml_transpose(h_bml,ht_bml) + call bml_transpose_new(h_bml,ht_bml) if(verbose.gt.0)then call bml_print_matrix("h_bml",h_bml,0,10,0,10) call bml_print_matrix("ht_bml",ht_bml,0,10,0,10) diff --git a/src/prg_nonortho_mod.F90 b/src/prg_nonortho_mod.F90 index 95a96119..9d5590bc 100644 --- a/src/prg_nonortho_mod.F90 +++ b/src/prg_nonortho_mod.F90 @@ -55,7 +55,7 @@ subroutine prg_orthogonalize(A_bml,zmat_bml,orthoA_bml,threshold,bml_type,verbos endif !Do the operations in bml - call bml_transpose(zmat_bml, aux_bml) + call bml_transpose_new(zmat_bml, aux_bml) call bml_multiply(aux_bml, A_bml, OrthoA_bml, 1.0_dp, 0.0_dp,threshold) !Z^t*A @@ -98,7 +98,7 @@ subroutine prg_deorthogonalize(orthoA_bml,zmat_bml,a_bml,threshold,bml_type,verb bml_get_distribution_mode(orthoA_bml)) endif - call bml_transpose(zmat_bml, aux_bml) + call bml_transpose_new(zmat_bml, aux_bml) call bml_multiply(orthoA_bml, aux_bml, a_bml, 1.0_dp, 0.0_dp, threshold) !orthoA*Z^t diff --git a/src/prg_pulaycomponent_mod.F90 b/src/prg_pulaycomponent_mod.F90 index 335bbb15..50a1a785 100644 --- a/src/prg_pulaycomponent_mod.F90 +++ b/src/prg_pulaycomponent_mod.F90 @@ -178,7 +178,7 @@ subroutine prg_get_pulayforce(nats,zmat_bml,ham_bml,rho_bml,& !SIHD = 2*Z*Z'*H*D; call bml_copy_new(zmat_bml,SIHD_bml) call bml_copy_new(zmat_bml,aux_bml) - call bml_transpose(zmat_bml,aux_bml) !Z' + call bml_transpose_new(zmat_bml,aux_bml) !Z' call bml_multiply(zmat_bml,aux_bml,SIHD_bml,2.0_dp,0.0_dp,threshold) !2*Z*Z' call bml_multiply(SIHD_bml,ham_bml,aux_bml,1.0_dp,0.0_dp,threshold) !2*Z*Z'*H call bml_multiply(aux_bml,rho_bml,SIHD_bml,1.0_dp,0.0_dp,threshold) !2*Z*Z'*D diff --git a/src/prg_response_mod.F90 b/src/prg_response_mod.F90 index f8165f4f..6fd4bb06 100644 --- a/src/prg_response_mod.F90 +++ b/src/prg_response_mod.F90 @@ -294,7 +294,7 @@ subroutine prg_compute_response_RS(ham_bml,prt_bml,rsp_bml,lambda& call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux1_bml) !Obtaining matrix $\f V = C^{\dagger} H^{(1)} C^{\dagger} $\f - call bml_transpose(umat_bml,aux_bml) !C^{\dagger} + call bml_transpose_new(umat_bml,aux_bml) !C^{\dagger} !First product C^{\dagger} H^{(1)} call bml_multiply(aux_bml,prt_bml,aux1_bml,1.0_dp,0.0_dp,threshold) !Seccond product to obtain V (aux_bml) @@ -340,11 +340,11 @@ subroutine prg_compute_response_RS(ham_bml,prt_bml,rsp_bml,lambda& !Cf(C^{(1)})^{t}$ call bml_zero_matrix(bml_type,bml_element_real,dp,norb,norb,aux2_bml) call bml_multiply(umat_bml,occupation_bml,aux_bml,1.0_dp,0.0_dp,threshold) - call bml_transpose(aux1_bml,aux2_bml) + call bml_transpose_new(aux1_bml,aux2_bml) call bml_multiply(aux_bml,aux2_bml,rsp_bml,1.0_dp,0.0_dp,threshold) !CfC^{(1)}$ + C^{(1)}fC - call bml_transpose(umat_bml,aux2_bml) + call bml_transpose_new(umat_bml,aux2_bml) call bml_multiply(occupation_bml,aux2_bml,aux_bml,1.0_dp,0.0_dp,threshold) call bml_deallocate(occupation_bml) diff --git a/src/prg_xlbokernel_mod.F90 b/src/prg_xlbokernel_mod.F90 index ed097543..5b862859 100644 --- a/src/prg_xlbokernel_mod.F90 +++ b/src/prg_xlbokernel_mod.F90 @@ -129,11 +129,11 @@ subroutine prg_kernel_multirank_latte(KRes,KK0_bml,Res,FelTol,L,LMAX,NUMRANK,HO_ call bml_zero_matrix(bml_type,bml_element_real,dp,N,MN,X_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,N,MN,Y_bml) - call bml_transpose(Z_bml,ZT_bml) + call bml_transpose_new(Z_bml,ZT_bml) ! K0Res = KK0*Res temporary for matrix-vector multiplication call bml_set_row(Res_bml,1,Res,1.0_dp*1e-10) - call bml_transpose(KK0_bml,KK0T_bml) + call bml_transpose_new(KK0_bml,KK0T_bml) call bml_multiply(Res_bml,KK0T_bml,K0Res_bml,1.0_dp,0.0_dp,1.0_dp*1e-10) call bml_get_row(K0Res_bml,1,row_NA) K0Res = row_NA @@ -298,7 +298,7 @@ subroutine prg_update_preconditioner(K0,v,fv,Nr_atoms,threshold) call bml_set_row(vt_bml,1,v,1.0_dp*1e-10) call bml_set_row(fv_bml,1,fv,1.0_dp*1e-10) - call bml_transpose(vt_bml,v_bml) + call bml_transpose_new(vt_bml,v_bml) call bml_multiply(K0,v_bml,tmp1_bml,1.0_dp,0.0_dp,threshold) call bml_copy(vt_bml,tmp2_bml) @@ -308,12 +308,12 @@ subroutine prg_update_preconditioner(K0,v,fv,Nr_atoms,threshold) call bml_multiply(onest_bml,K0,tmp1_bml,1.0_dp,0.0_dp,0.0_dp) call bml_get_row(tmp1_bml,1,norm) - call bml_transpose(K0,K0_T) + call bml_transpose_new(K0,K0_T) do I = 1,Nr_atoms call bml_get_row(K0_T,1,row) call bml_set_row(K0_T,1,row/norm(I),1.0_dp*1e-10) enddo - call bml_transpose(K0_T,K0) + call bml_transpose_new(K0_T,K0) deallocate(row) deallocate(norm) @@ -376,7 +376,7 @@ subroutine prg_kernel_multirank(KRes,KK0_bml,Res,FelTol,L,LMAX,HO_bml,mu,beta,RX call bml_zero_matrix("ellpack",bml_element_real,dp,Nr_atoms,Nr_atoms,Res_bml) ! K0Res = KK0*Res temporary for matrix-vector multiplication - call bml_transpose(KK0_bml,KK0T_bml) + call bml_transpose_new(KK0_bml,KK0T_bml) call bml_set_row(Res_bml,1,Res,ONE*1e-14) call bml_multiply(Res_bml,KK0T_bml,K0Res_bml,1.0_dp,0.0_dp,ONE*1e-14) call bml_get_row(K0Res_bml,1,row_NA) @@ -425,14 +425,14 @@ subroutine prg_kernel_multirank(KRes,KK0_bml,Res,FelTol,L,LMAX,HO_bml,mu,beta,RX call bml_multiply(H1_bml,S_bml,D1_bml,0.5_dp,0.5_dp,threshold) call bml_copy(D1_bml,H1_bml) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(X_bml,H1_bml,D1_bml,1.0_dp,0.0_dp,threshold) call bml_multiply(D1_bml,Z_bml,H1_bml,1.0_dp,0.0_dp,threshold) call prg_implicit_fermi_first_order_response(HO_bml,H1_bml,DO_bml,D1_bml,Inv_bml, & m_rec, mu, beta, real(nocc,dp), threshold) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(Z_bml,D1_bml,Y_bml,2.0_dp,0.0_dp,threshold) call bml_multiply(Y_bml,X_bml,D1_bml,1.0_dp,0.0_dp,threshold) @@ -547,7 +547,7 @@ subroutine prg_full_kernel_latte(KK,DO_bml,mu0,RXYZ,Box,Hubbard_U, & call bml_zero_matrix(bml_type,bml_element_real,dp,N,MN,ZT_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,N,MN,X_bml) call bml_zero_matrix(bml_type,bml_element_real,dp,N,MN,Y_bml) - call bml_transpose(Z_bml,ZT_bml) + call bml_transpose_new(Z_bml,ZT_bml) allocate(row1(HDIM)); allocate(row2(HDIM)); allocate(JJ(Nr_atoms,Nr_atoms)) allocate(EReal(Nr_atoms,Nr_atoms)); allocate(EKspace(Nr_atoms,Nr_atoms)) allocate(E(Nr_atoms,Nr_atoms)) @@ -750,14 +750,14 @@ subroutine prg_full_kernel(KK,DO_bml,mu0,RX,RY,RZ,LBox,Hubbard_U,Element_Type, & call bml_multiply(H1_bml,S_bml,D1_bml,0.5_dp,0.5_dp,threshold) call bml_copy(D1_bml,H1_bml) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(X_bml,H1_bml,D1_bml,1.0_dp,0.0_dp,threshold) call bml_multiply(D1_bml,Z_bml,H1_bml,1.0_dp,0.0_dp,threshold) call prg_implicit_fermi_first_order_response(HO_bml,H1_bml,DO_bml,D1_bml,Inv_bml, & m_rec,mu0,beta,real(nocc,PREC),threshold) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(Z_bml,D1_bml,Y_bml,2.0_dp,0.0_dp,threshold) call bml_multiply(Y_bml,X_bml,D1_bml,1.0_dp,0.0_dp,threshold) @@ -867,14 +867,14 @@ subroutine prg_kernel_matrix_multirank(KRes,KK0_bml,Res,FelTol,L,LMAX,HO_bml,mu, call bml_multiply(H1_bml,S_bml,D1_bml,0.5_dp,0.5_dp,threshold) call bml_copy(D1_bml,H1_bml) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(X_bml,H1_bml,D1_bml,1.0_dp,0.0_dp,threshold) call bml_multiply(D1_bml,Z_bml,H1_bml,1.0_dp,0.0_dp,threshold) call prg_implicit_fermi_first_order_response(HO_bml,H1_bml,DO_bml,D1_bml,Inv_bml, & m_rec, mu, beta, Nocc, threshold) - call bml_transpose(Z_bml,X_bml) + call bml_transpose_new(Z_bml,X_bml) call bml_multiply(Z_bml,D1_bml,Y_bml,2.0_dp,0.0_dp,threshold) call bml_multiply(Y_bml,X_bml,D1_bml,1.0_dp,0.0_dp,threshold) From 6195172627b0edad6ef1a3ce08566f75c2db4fbb Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 24 Jul 2024 12:53:42 -0600 Subject: [PATCH 23/69] Add nvtx tags for charges and thread charge calculation --- bml | 2 +- examples/gpmdk/src/gpmdcov_kernel.F90 | 18 ++++++++++++++++++ src/prg_charges_mod.F90 | 7 +++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bml b/bml index 43cc151d..d2d85933 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 43cc151d873069e299314a5e08d117c8ec6f1169 +Subproject commit d2d8593313caccf9ea7933e76d2dc9db718bd6bd diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index aefad0ca..7069ca89 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -1069,6 +1069,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !a vector q1 that stores all the previous q1s from past iranks iterations !We also compute the partial trace contribution (trP1) from this mpi !execution and the current part (ipt). +#ifdef USE_NVTX + call nvtxStartRange("Charges",5) +#endif ptnet_charge = 0.0_dp mynumel = 0.0_dp call prg_get_charges(p1_bml, mysyprt(ipt)%estr%over,& @@ -1092,6 +1095,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !call bml_deallocate(dPdMu_bml) call bml_deallocate(ptaux_bml) deallocate(ptnet_charge) +#ifdef USE_NVTX + call nvtxEndRange +#endif enddo !End of the loop over the parts @@ -1105,6 +1111,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re mu1_Global = -trP1(1)/trdPdMu(1) q1 = q1 + mu1_Global*dqdmu +#ifdef USE_NVTX + call nvtxStartRange("dr",6) +#endif call gpmdcov_reallocate_realVect(f,maxCoresAmongPartsAndRanks) !deallocate(maxCoresAmongParts) @@ -1134,6 +1143,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re endif #endif dr_save(:,iRank) = dr +#ifdef USE_NVTX + call nvtxEndRange +#endif enddo !Rank enddo @@ -1143,6 +1155,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call prg_sumRealReduceN(c_i, mRanks) endif #endif +#ifdef USE_NVTX + call nvtxStartRange("Calculate V",7) +#endif allocate(auxVect(mRanks*mRanks)) auxVect = 0.0_dp @@ -1215,6 +1230,9 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re ! write(*,*) DOT_PRODUCT(dr_save(:,myi),dr_save(:,myj)) ! enddo ! enddo +#ifdef USE_NVTX + call nvtxEndRange +#endif #ifdef USE_NVTX call nvtxStartRange("Deallocations",5) #endif diff --git a/src/prg_charges_mod.F90 b/src/prg_charges_mod.F90 index a5cac108..9f723e8e 100644 --- a/src/prg_charges_mod.F90 +++ b/src/prg_charges_mod.F90 @@ -66,7 +66,9 @@ subroutine prg_get_charges(rho_bml,over_bml,hindex,charges,numel,spindex,mdimin, #endif call bml_get_diagonal(aux_bml,rho_diag) - + !$omp parallel do default(none) private(i) & + !$omp private(j,znuc) & + !$omp shared(spindex,numel,charges,hindex,rho_diag,nats) do i = 1,nats znuc = numel(spindex(i)) charges(i)=0.0_dp @@ -74,7 +76,8 @@ subroutine prg_get_charges(rho_bml,over_bml,hindex,charges,numel,spindex,mdimin, charges(i) = charges(i) + rho_diag(j) enddo charges(i) = charges(i) - znuc - enddo + enddo + !$omp end parallel do deallocate(rho_diag) call bml_deallocate(aux_bml) From 2c72a4019d2e8a2b06f09f32e4ff7b1ad77b18a4 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 24 Jul 2024 13:17:31 -0600 Subject: [PATCH 24/69] Add nvtx tag methods to gpmdk --- examples/gpmdk/src/gpmdcov_nvtx.F90 | 63 +++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 examples/gpmdk/src/gpmdcov_nvtx.F90 diff --git a/examples/gpmdk/src/gpmdcov_nvtx.F90 b/examples/gpmdk/src/gpmdcov_nvtx.F90 new file mode 100644 index 00000000..cfa2f716 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_nvtx.F90 @@ -0,0 +1,63 @@ +module gpmdcov_nvtx_mod + use iso_c_binding + implicit none + + integer(4),private :: col(7) = [ int(Z'0000ff00',4), int(Z'000000ff',4), int(Z'00ffff00',4), int(Z'00ff00ff',4), int(Z'0000ffff',4), int(Z'00ff0000',4), int(Z'00ffffff',4)] + character(kind=c_char,len=256),private,target :: tempName + + type, bind(C):: nvtxEventAttributes + integer(C_INT16_T):: version=1 + integer(C_INT16_T):: size=48 ! + integer(C_INT):: category=0 + integer(C_INT):: colorType=1 ! NVTX_COLOR_ARGB = 1 + integer(C_INT):: color + integer(C_INT):: payloadType=0 ! NVTX_PAYLOAD_UNKNOWN = 0 + integer(C_INT):: reserved0 + integer(C_INT64_T):: payload ! union uint,int,double + integer(C_INT):: messageType=1 ! NVTX_MESSAGE_TYPE_ASCII = 1 + type(C_PTR):: message ! ascii char + end type nvtxEventAttributes +#ifdef USE_NVTX + interface nvtxRangePush + ! push range with custom label and standard color + subroutine nvtxRangePushA(name) bind(C, name='nvtxRangePushA') + use iso_c_binding + character(kind=C_CHAR,len=*) :: name + end subroutine nvtxRangePushA + + ! push range with custom label and custom color + subroutine nvtxRangePushEx(event) bind(C, name='nvtxRangePushEx') + use iso_c_binding + import:: nvtxEventAttributes + type(nvtxEventAttributes):: event + end subroutine nvtxRangePushEx + end interface nvtxRangePush + + interface nvtxRangePop + subroutine nvtxRangePop() bind(C, name='nvtxRangePop') + end subroutine nvtxRangePop + end interface nvtxRangePop + +contains + + subroutine nvtxStartRange(name,id) + character(kind=c_char,len=*) :: name + integer, optional:: id + type(nvtxEventAttributes):: event + + tempName=trim(name)//c_null_char + write(*,*)"gpmdcov_nvtx"," Tag = "//name + if ( .not. present(id)) then + call nvtxRangePush(tempName) + else + event%color=col(mod(id,7)+1) + event%message=c_loc(tempName) + call nvtxRangePushEx(event) + end if + end subroutine nvtxStartRange + + subroutine nvtxEndRange + call nvtxRangePop + end subroutine nvtxEndRange +#endif +end module gpmdcov_nvtx_mod From 2c7afd7b3f223179785d3d07f42930c68b9da41a Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Thu, 25 Jul 2024 14:52:46 -0600 Subject: [PATCH 25/69] Debug cray build and work on omp offload --- examples/gpmdk/src/gpmdcov_response.F90 | 50 +++++++++++++++++++------ src/latte_mods/ham_latte_mod.F90 | 30 +++++++-------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index a06b6e33..e6bbabad 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -74,13 +74,21 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 call bml_export_to_dense(P1_bml,P1) call bml_export_to_dense(DX1_bml,DX1) -!!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) - +!!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) +#ifdef USE_OFFLOAD + !$omp target enter data map(alloc:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) + !$omp target update to(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) +#endif do i = 1,m ! Loop over m recursion steps - !p_02 = p_0*p_0 - !$omp parallel do default(none) private(k) & - !$omp private(j) & - !$omp firstprivate(p_0) & + !p_02 = p_0*p_0 +#ifdef USE_OFFLOAD + !$omp target teams distribute default(none) & + !$omp shared(p_0) & +#else + !$omp parallel do default(none) & + !$omp firstprivate(p_0) & +#endif + !$omp private(k) & !$omp shared(HDIM,P1,DX1) !!$acc parallel loop deviceptr(P1,DX1,p_0) !!$acc parallel loop @@ -89,28 +97,46 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !DX1(k,:) = (p_0(k) + p_0(:))*P1(k,:) enddo !!$acc end parallel loop - !$omp end parallel do !iD0 = 1.D0/(2.D0*(p_0*p_0-p_0)+1.D0) - - !$omp parallel do default(none) private(k) & - !$omp private(j) & - !$omp firstprivate(iD0,p_0) & +#ifdef USE_OFFLOAD + !$omp end target teams distribute + !$omp target teams distribute default(none) & + !$omp shared(p_0) & +#else + !$omp end parallel do + !$omp parallel do default(none) & + !$omp firstprivate(p_0) & +#endif + !$omp private(k) & !$omp shared(HDIM,P1,DX1) !!$acc parallel loop deviceptr(P1,DX1,p_0) !!$acc parallel loop do k = 1,HDIM - P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) + P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) + !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) enddo !!$acc end parallel loop +#ifdef USE_OFFLOAD + !$omp end target teams distribute + !$omp target +#else !$omp end parallel do +#endif !!$acc kernels deviceptr(p_0) !!$acc kernels p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) +#ifdef USE_OFFLOAD + !$omp end target +#endif !!$acc end kernels enddo !!$acc end data +#ifdef USE_OFFLOAD + !$omp target update from(P1(1:HDIM,1:HDIM)) + !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) +#endif bml_type = bml_get_type(P1_bml) call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml diff --git a/src/latte_mods/ham_latte_mod.F90 b/src/latte_mods/ham_latte_mod.F90 index 1d7cd8ad..56f934e6 100644 --- a/src/latte_mods/ham_latte_mod.F90 +++ b/src/latte_mods/ham_latte_mod.F90 @@ -154,7 +154,7 @@ subroutine get_hsmat(ham_bml,over_bml,coordinate,lattice_vector,spindex,& integer, intent(in) :: hindex(:,:), norbi(:), spindex(:) integer :: maxnorbi real(dp) :: ra(3), rb(3) - real(dp), allocatable :: block(:,:,:), ham(:,:), over(:,:) + real(dp), allocatable :: blk(:,:,:), ham(:,:), over(:,:) real(dp), intent(in) :: coordinate(:,:), lattice_vector(:,:), onsitesH(:,:), onsitesS(:,:) real(dp), intent(in) :: threshold type(bml_matrix_t), intent(inout) :: ham_bml, over_bml @@ -184,14 +184,14 @@ subroutine get_hsmat(ham_bml,over_bml,coordinate,lattice_vector,spindex,& allocate(over(norb,norb)) endif - if (.not.allocated(block)) then - allocate(block(maxnorbi,maxnorbi,nats)) + if (.not.allocated(blk)) then + allocate(blk(maxnorbi,maxnorbi,nats)) endif !$omp parallel do default(none) & - !$omp private(ra,rb,dimi,dimj,ii,jj,j) & + !$omp private(i,ra,rb,dimi,dimj,ii,jj,j) & !$omp shared(nats,coordinate,hindex,spindex, intPairsS,intPairsH,threshold,lattice_vector,norbi,onsitesH,onsitesS,ham_bml,over_bml) & - !$omp shared(block,ham,over) + !$omp shared(blk,ham,over) do i = 1, nats ra(:) = coordinate(:,i) dimi = hindex(2,i)-hindex(1,i)+1 @@ -201,27 +201,26 @@ subroutine get_hsmat(ham_bml,over_bml,coordinate,lattice_vector,spindex,& !Hamiltonian block for a-b atom pair call get_SKBlock(spindex(i),spindex(j),coordinate(:,i),& coordinate(:,j),lattice_vector,norbi,& - onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,block,i) + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blk,i) do jj=1,dimj do ii=1,dimi - ham(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = block(ii,jj,i) + ham(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) enddo enddo call get_SKBlock(spindex(i),spindex(j),coordinate(:,i),& coordinate(:,j),lattice_vector,norbi,& - onsitesS,intPairsS(spindex(i),spindex(j))%intParams,intPairsS(spindex(j),spindex(i))%intParams,block,i) - + onsitesS,intPairsS(spindex(i),spindex(j))%intParams,intPairsS(spindex(j),spindex(i))%intParams,blk,i) do jj=1,dimj do ii=1,dimi - over(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = block(ii,jj,i) + over(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) enddo - enddo + enddo ! write(*,*)spindex(i),spindex(j) ! write(*,'(100F10.5)')intPairsS(spindex(i),spindex(j))%intParams - ! call prg_print_matrix("block",block(:,:,i),1,4,1,4) + ! call prg_print_matrix("blk",blk(:,:,i),1,4,1,4) enddo enddo @@ -564,10 +563,9 @@ subroutine get_SKBlock(sp1,sp2,coorda,coordb,lattice_vectors& !rb(3) = RZb + nr_shift_z*lattice_vectors(3,3) ! shifts for pbc ! rb(3) = rb(3) + nr_shift_y*lattice_vectors(2,3) ! shifts for pbc ! rb(3) = rb(3) + nr_shift_x*lattice_vectors(1,3) ! shifts for pbc - do i = 1,3 - Rab(i) = modulo((Rb(i) - Ra(i)) + 0.5_dp*lattice_vectors(i,i),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) - enddo - + do i = 1,3 + Rab(i) = modulo((Rb(i)-Ra(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) + enddo !Rab = Rb-Ra; ! OBS b - a !!! !dR = sqrt(Rab(1)**2+ Rab(2)**2+ Rab(3)**2) dR = norm2(Rab) From 0ae84e1c66bc7b15b96e65c883dbacc5cdb44ec7 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 29 Jul 2024 10:30:35 -0600 Subject: [PATCH 26/69] Working omp offload for gpmdcov_response --- examples/gpmdk/src/gpmdcov_response.F90 | 83 +++++++++++++------------ 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index e6bbabad..65b92ed3 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -10,6 +10,7 @@ module gpmdcov_response_mod use prg_kernelparser_mod use prg_densitymatrix_mod use prg_openfiles_mod + use gpmdcov_nvtx_mod implicit none @@ -61,81 +62,81 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 h_0 = evals ! Diagonal Hamiltonian H0 respresented in the eigenbasis Q cnst = beta/(1.D0*2**(m+2)) ! Scaling constant p_0 = 0.5D0 + cnst*(h_0-mu0) ! Initialization for P0 represented in eigenbasis Q +#ifdef USE_NVTX + call nvtxStartRange("bml_copy_new",1) +#endif call bml_copy_new(H1_bml,P1_bml) call bml_copy_new(H1_bml,DX1_bml) +#ifdef USE_NVTX + call nvtxEndRange +#endif call bml_scale(-cnst,P1_bml) !(set mu1 = 0 for simplicity) !Initialization of DM response in Q representation (not diagonal in Q) - allocate(P1(HDIM,HDIM)) allocate(DX1(HDIM,HDIM)) P1 = 0.0_dp DX1 = 0.0_dp +#ifdef USE_NVTX + call nvtxStartRange("bml_export_to_dense",2) +#endif call bml_export_to_dense(P1_bml,P1) call bml_export_to_dense(DX1_bml,DX1) - +#ifdef USE_NVTX + call nvtxEndRange +#endif + !!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) #ifdef USE_OFFLOAD - !$omp target enter data map(alloc:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) - !$omp target update to(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) + !$omp target enter data map(alloc:P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) + !$omp target update to(P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM) +#endif +#ifdef USE_NVTX + call nvtxStartRange("Response Kernel",3) #endif do i = 1,m ! Loop over m recursion steps !p_02 = p_0*p_0 #ifdef USE_OFFLOAD !$omp target teams distribute default(none) & - !$omp shared(p_0) & -#else - !$omp parallel do default(none) & - !$omp firstprivate(p_0) & -#endif - !$omp private(k) & - !$omp shared(HDIM,P1,DX1) - !!$acc parallel loop deviceptr(P1,DX1,p_0) - !!$acc parallel loop - do k = 1,HDIM - DX1(:,k) = (p_0(:) + p_0(k))*P1(:,k) - !DX1(k,:) = (p_0(k) + p_0(:))*P1(k,:) - enddo - !!$acc end parallel loop - !iD0 = 1.D0/(2.D0*(p_0*p_0-p_0)+1.D0) -#ifdef USE_OFFLOAD - !$omp end target teams distribute - !$omp target teams distribute default(none) & - !$omp shared(p_0) & -#else - !$omp end parallel do - !$omp parallel do default(none) & - !$omp firstprivate(p_0) & -#endif - !$omp private(k) & - !$omp shared(HDIM,P1,DX1) !!$acc parallel loop deviceptr(P1,DX1,p_0) !!$acc parallel loop - do k = 1,HDIM - P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) + !$omp shared(HDIM,P1,p_0) + do k = 1,HDIM + !$omp parallel do + do j = 1,HDIM + P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + enddo + !$omp end parallel do !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) enddo !!$acc end parallel loop -#ifdef USE_OFFLOAD + !!$acc kernels deviceptr(p_0) + !!$acc kernels !$omp end target teams distribute !$omp target + p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) + !$omp end target #else + !$omp parallel do default(none) & + !$omp private(k) & + !$omp shared(HDIM,P1,p_0) + do k = 1,HDIM + P1(:,k) = 1.D0/(2.D0*p_0(:)*(p_0(:)-1.D0)+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) + !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) + enddo !$omp end parallel do -#endif - !!$acc kernels deviceptr(p_0) - !!$acc kernels p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) -#ifdef USE_OFFLOAD - !$omp end target #endif - !!$acc end kernels enddo - +#ifdef USE_NVTX + call nvtxEndRange +#endif !!$acc end data #ifdef USE_OFFLOAD !$omp target update from(P1(1:HDIM,1:HDIM)) - !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) + !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) #endif bml_type = bml_get_type(P1_bml) From 468bca180af48d49c5755f75528b73b23980d597 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 30 Jul 2024 11:41:37 -0600 Subject: [PATCH 27/69] Bug fix --- examples/gpmdk/src/gpmdcov_kernel.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 7069ca89..bc63eb49 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -995,12 +995,12 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,maxorbs) call gpmdcov_reallocate_denseBmlRealMat(zq_bml,maxorbs) call gpmdcov_reallocate_denseBmlRealMat(zqt_bml,maxorbs) - call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) + !call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) endif call gpmdcov_bml_set_N(ptham_bml,norbs) call gpmdcov_bml_set_N(zq_bml,norbs) call gpmdcov_bml_set_N(zqt_bml,norbs) - call gpmdcov_bml_set_N(dPdMu_bml,norbs) + !call gpmdcov_bml_set_N(dPdMu_bml,norbs) ! call gpmdcov_reallocate_denseBmlRealMat(ptham_bml,norbs) #ifdef USE_NVTX call nvtxEndRange @@ -1052,7 +1052,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re call bml_multiply(ptaux_bml,zqt_bml,p1_bml,2.0_dp,0.0_dp,0.0_dp) !Since dPdMu is represented by a vector, we will convert it into a BML matrix - !call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) + call gpmdcov_reallocate_denseBmlRealMat(dPdMu_bml,norbs) call bml_set_diagonal(dPdMu_bml,dPdMu) !Transform dPdMu back to the nonortho-canonical basis set @@ -1092,7 +1092,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re mysyprt(ipt)%spindex, norbs, lt%threshold) dqdmu(:,iptt) = ptnet_charge(1:natsCore) trdPdMu = trdPdMu + sum(ptnet_charge(1:natsCore)) - !call bml_deallocate(dPdMu_bml) + call bml_deallocate(dPdMu_bml) call bml_deallocate(ptaux_bml) deallocate(ptnet_charge) #ifdef USE_NVTX From 07f9dcb8fbe8b35f594bd92c970e3158c01aec63 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 30 Jul 2024 16:22:45 -0600 Subject: [PATCH 28/69] Allocate smaller array for work in kernel --- examples/gpmdk/src/gpmdcov_kernel.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index bc63eb49..8c320732 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -913,7 +913,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re ff = 0.0_dp if (maxorbs == -1) then - maxorbs = min(maxCoresAmongPartsAndRanks*20,sy%nats)*4 + maxorbs = min(maxval(gpat%sgraph(:)%lsize)*3/2,sy%nats)*4 endif !Here we enter the loop for the rank updates (do not confuse with MPI rank) From 5edf99b2b1e413f0f4bd8db67215e27a9ccc4606 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 6 Aug 2024 09:27:27 -0600 Subject: [PATCH 29/69] Code decorations to investigate MPI imbalance --- build_venado_hackathon.sh | 4 ++-- examples/gpmdk/build_venado_hackathon.sh | 4 ++-- examples/gpmdk/build_venado_hackathon_nvtx.sh | 2 +- examples/gpmdk/run/TrpCage/sbatch.sh | 2 +- examples/gpmdk/run/TrpCage/sbatch_profile.sh | 4 ++-- examples/gpmdk/run/TrpCage/wrapper_profile.sh | 2 +- examples/gpmdk/src/gpmdcov_constraints.F90 | 2 +- examples/gpmdk/src/gpmdcov_dm_min.F90 | 12 ++++++++++++ examples/gpmdk/src/gpmdcov_energandforces.F90 | 7 ++++--- examples/gpmdk/src/gpmdcov_initparts.F90 | 18 ++++++++++++++++++ examples/gpmdk/src/gpmdcov_kernel.F90 | 1 + examples/gpmdk/src/gpmdcov_neighbor.F90 | 2 +- examples/gpmdk/src/gpmdcov_response.F90 | 11 ++++++++--- scripts/setenv_venado.sh | 2 +- 14 files changed, 55 insertions(+), 18 deletions(-) diff --git a/build_venado_hackathon.sh b/build_venado_hackathon.sh index 697ef254..d1541b79 100755 --- a/build_venado_hackathon.sh +++ b/build_venado_hackathon.sh @@ -17,7 +17,7 @@ MY_PATH=`pwd` export CC=${CC:=cc} export FC=${FC:=ftn} export CXX=${CXX:=CC} -export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ffixed-line-length-none"} +#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ffixed-line-length-none"} #export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"} export PKG_CONFIG_PATH="$BML_LIB/lib/pkgconfig:$BML_LIB/lib64/pkgconfig:$PKG_CONFIG_PATH" export PROGRESS_OPENMP=${PROGRESS_OPENMP:=yes} @@ -30,7 +30,7 @@ export PROGRESS_EXAMPLES=${PROGRESS_EXAMPLES:=yes} #export PROGRESS_BENCHMARKS=yes export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=$METIS_LIB/:$BML_LIB/} #export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch -fdefault-integer-8"} -export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} +export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-O3 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch -fopt-info -mcpu=neoverse-v2"} #export EXTRA_FCFLAGS=${EXTRA_FCFLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} #export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-g -O2 -ffree-line-length-none -Wno-pedantic -fallow-argument-mismatch"} export CMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH:="$METIS_LIB/include"} diff --git a/examples/gpmdk/build_venado_hackathon.sh b/examples/gpmdk/build_venado_hackathon.sh index 0aeb42e6..d57fb62d 100755 --- a/examples/gpmdk/build_venado_hackathon.sh +++ b/examples/gpmdk/build_venado_hackathon.sh @@ -2,8 +2,8 @@ WORKDIR=/usr/projects/icapt/mewall/venado/packages rm -rf build_hackathon mkdir build_hackathon cd build_hackathon -cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" \ --DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero" \ +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ +-DEXTRA_FCFLAGS="-O3 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt -ftree-vectorizer-verbose=2 -mcpu=neoverse-v2" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ #-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ make -j1 diff --git a/examples/gpmdk/build_venado_hackathon_nvtx.sh b/examples/gpmdk/build_venado_hackathon_nvtx.sh index 72391d0f..d57fb62d 100755 --- a/examples/gpmdk/build_venado_hackathon_nvtx.sh +++ b/examples/gpmdk/build_venado_hackathon_nvtx.sh @@ -3,7 +3,7 @@ rm -rf build_hackathon mkdir build_hackathon cd build_hackathon cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" -DLIB="no" -DGPMDK_NVTX="yes" \ --DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ +-DEXTRA_FCFLAGS="-O3 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt -ftree-vectorizer-verbose=2 -mcpu=neoverse-v2" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/install" ../src/ #-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ make -j1 diff --git a/examples/gpmdk/run/TrpCage/sbatch.sh b/examples/gpmdk/run/TrpCage/sbatch.sh index 0d0710b3..ce6b3672 100755 --- a/examples/gpmdk/run/TrpCage/sbatch.sh +++ b/examples/gpmdk/run/TrpCage/sbatch.sh @@ -15,5 +15,5 @@ source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_ve export MPICH_SMP_SINGLE_COPY_MODE=NONE #export MPICH_OPT_THREAD_SYNC=0 -OMP_NUM_THREADS=72 srun -n 256 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh +OMP_NUM_THREADS=72 srun -n 64 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh diff --git a/examples/gpmdk/run/TrpCage/sbatch_profile.sh b/examples/gpmdk/run/TrpCage/sbatch_profile.sh index 77ac0576..ac2f83d9 100755 --- a/examples/gpmdk/run/TrpCage/sbatch_profile.sh +++ b/examples/gpmdk/run/TrpCage/sbatch_profile.sh @@ -11,8 +11,8 @@ cd ${PWD} source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh -#export MPICH_ALLREDUCE_NO_SMP=1 +export MPICH_ALLREDUCE_NO_SMP=1 export MPICH_SMP_SINGLE_COPY_MODE=NONE -OMP_NUM_THREADS=72 srun -n 256 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper_profile.sh +OMP_NUM_THREADS=72 srun -n 64 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper_profile.sh diff --git a/examples/gpmdk/run/TrpCage/wrapper_profile.sh b/examples/gpmdk/run/TrpCage/wrapper_profile.sh index e380fb4d..f5518c82 100644 --- a/examples/gpmdk/run/TrpCage/wrapper_profile.sh +++ b/examples/gpmdk/run/TrpCage/wrapper_profile.sh @@ -7,7 +7,7 @@ RUN="/usr/projects/icapt/mewall/venado/packages/qmd-progress/examples/gpmdk/buil #stdbuf -o0 $RUN input_ICH.in |& tee out_ICH if [ $SLURM_PROCID -eq 0 ]; then # nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -t mpi,openmp,cuda,nvtx --mpi-impl=mpich --delay=90 $RUN input.in >& out_0 - nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -c cudaProfilerApi -t mpi,openmp,cuda,nvtx --mpi-impl=mpich -o report_%q{SLURM_PROCID} $RUN input.in >& out_$SLURM_PROCID + nsys profile -e NSYS_MPI_STORE_TEAMS_PER_RANK=1 -c cudaProfilerApi -t mpi,openmp,cuda,nvtx --mpi-impl=mpich $RUN input.in >& out_$SLURM_PROCID #nsys profile -t mpi,openmp,cuda,nvtx --mpi-impl=mpich --delay=90 -o report_%q{SLURM_PROCID} $RUN input.in >& out_$SLURM_PROCID else $RUN input.in >& /dev/null diff --git a/examples/gpmdk/src/gpmdcov_constraints.F90 b/examples/gpmdk/src/gpmdcov_constraints.F90 index 8eec5a1a..ac41faf5 100644 --- a/examples/gpmdk/src/gpmdcov_constraints.F90 +++ b/examples/gpmdk/src/gpmdcov_constraints.F90 @@ -46,7 +46,7 @@ subroutine gpmdcov_constraint_harmonicToLinear(atom1r, atom2r, force_total, ener !! Convert to fractional coordinates by dividing by box dimensions !! Compute fractional distances do k = 1,3 - dcoords(k) = modulo((atom1r(k) - atom2r(k)) + 0.5_dp*Lbox(k),Lbox(k)) - 0.5_dp * Lbox(k) + dcoords(k) = modulo(((atom1r(k) - atom2r(k)) + 0.5_dp*Lbox(k)),Lbox(k)) - 0.5_dp * Lbox(k) enddo r_separation = norm2(dcoords) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index a7b077bf..c43a0575 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -1,6 +1,10 @@ module gpmdcov_DM_Min_mod use gpmdcov_nonequilibrium_mod +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif + contains !> SCF loop @@ -328,6 +332,14 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before gpmdcov_muFromParts",lt%verbose,myRank) + +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierAfterH1",2) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif if(myRank == 1 .and. lt%verbose >= 1) mls_mu = mls() call gpmdcov_muFromParts() call gpmdcov_msII("gpmdcov_DM_Min","Time for get Mu "//to_string(mls() - mls_mu)//" ms",lt%verbose,myRank) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 0ba912da..d34fa108 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -210,8 +210,8 @@ subroutine gpmdcov_EnergAndForces(charges) call gpmdcov_msI("gpmdcov_EnergAndForces","SMD Total Energy " & & // to_string(smd_total_energy),lt%verbose,myRank) do k = 1,3 - dcoords(k) = modulo((R1(k) - R2(k)) + & - &0.5_dp*sy%lattice_vector(k,k),sy%lattice_vector(k,k)) - & + dcoords(k) = modulo(((R1(k) - R2(k)) + & + &0.5_dp*sy%lattice_vector(k,k)),sy%lattice_vector(k,k)) - & &0.5_dp * sy%lattice_vector(k,k) enddo dist = norm2(dcoords) @@ -272,7 +272,8 @@ subroutine gpmdcov_EnergAndForces(charges) if (getNRanks() .gt. 1) then ! call prg_sumRealReduceN(collectedforce(1,:), sy%nats) ! call prg_sumRealReduceN(collectedforce(2,:), sy%nats) - ! call prg_sumRealReduceN(collectedforce(3,:), sy%nats) + ! call prg_sumRealReduceN(collectedforce(3,:), sy%nats) + call prg_barrierParallel call prg_sumRealReduceN(collectedforce, sy%nats*3) call prg_sumRealReduceN(ebandvector, gpat%TotalParts) endif diff --git a/examples/gpmdk/src/gpmdcov_initparts.F90 b/examples/gpmdk/src/gpmdcov_initparts.F90 index f0e71c32..e56e57ab 100644 --- a/examples/gpmdk/src/gpmdcov_initparts.F90 +++ b/examples/gpmdk/src/gpmdcov_initparts.F90 @@ -6,6 +6,10 @@ subroutine gpmdcov_InitParts #ifdef USE_LATTE use gpmdcov_latte_mod #endif +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif + implicit none integer :: norbsCore @@ -60,6 +64,13 @@ subroutine gpmdcov_InitParts call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%ham0) call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%over) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierBeforeGetHS",2) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif !Construction of the hamiltonian for every part #ifdef USE_LATTE @@ -87,6 +98,13 @@ subroutine gpmdcov_InitParts ! stop endif endif +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierAfterGetHS",3) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif !> Get occupation based on last shell population. ! WARNING: This could change depending on the TB method being used. diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 8c320732..845b5397 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -1104,6 +1104,7 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re !We will gather all the partial traces from all the MPI Ranks. #ifdef DO_MPI if (getNRanks() .gt. 1) then + call prg_barrierParallel call prg_sumRealReduceN(trP1, 1) call prg_sumRealReduceN(trdPdMu, 1) endif diff --git a/examples/gpmdk/src/gpmdcov_neighbor.F90 b/examples/gpmdk/src/gpmdcov_neighbor.F90 index b48faaee..9246a923 100644 --- a/examples/gpmdk/src/gpmdcov_neighbor.F90 +++ b/examples/gpmdk/src/gpmdcov_neighbor.F90 @@ -180,7 +180,7 @@ subroutine gpmdcov_build_nlist_full(coords,lattice_vectors,rcut,nl,verbose,rank, do i = 1,nats !For every atom #endif do k = 1,3 - fdvarray(:,k) = modulo((fcoords(:,k) - fcoords(i,k)) + 0.5,1.) - 0.5 + fdvarray(:,k) = modulo(((fcoords(:,k) - fcoords(i,k)) + 0.5_dp),1._dp) - 0.5_dp enddo dvarray = matmul(fdvarray,lattice_vectors) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 65b92ed3..8d3016e9 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -119,10 +119,15 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !$omp end target #else !$omp parallel do default(none) & - !$omp private(k) & + !$omp private(k,j) & !$omp shared(HDIM,P1,p_0) - do k = 1,HDIM - P1(:,k) = 1.D0/(2.D0*p_0(:)*(p_0(:)-1.D0)+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + do k = 1,HDIM + !$omp simd + do j = 1,HDIM + !P1(:,k) = 1.D0/(2.D0*p_0(:)*(p_0(:)-1.D0)+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + enddo + !$omp end simd !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) enddo diff --git a/scripts/setenv_venado.sh b/scripts/setenv_venado.sh index 09f1c773..49271414 100644 --- a/scripts/setenv_venado.sh +++ b/scripts/setenv_venado.sh @@ -1,3 +1,3 @@ module switch PrgEnv-cray PrgEnv-gnu -export PKG_CONFIG_PATH=/opt/cray/xpmem/2.8.2-1.0_5.1__g84a27a5.shasta/lib64/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH=/opt/cray/xpmem/2.8.4-1.0_7.1__ga37cbd9.shasta/lib64/pkgconfig:$PKG_CONFIG_PATH module load cudatoolkit From b941d1712d1364f2ebb3105630d198cc2055b706 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 7 Aug 2024 14:58:09 -0600 Subject: [PATCH 30/69] Build updates --- examples/gpmdk/build_chicoma_hackathon.sh | 4 +-- examples/gpmdk/run/TrpCage/sbatch.sh | 14 ++++++---- examples/gpmdk/run/TrpCage/wrapper.sh | 2 +- examples/gpmdk/run/water/input.in | 12 ++++----- examples/gpmdk/run/water/sbatch.sh | 4 +-- examples/gpmdk/src/gpmdcov_part.F90 | 31 ++++++++++++++++++++--- 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/examples/gpmdk/build_chicoma_hackathon.sh b/examples/gpmdk/build_chicoma_hackathon.sh index b04c9c4b..bfd2f5e0 100755 --- a/examples/gpmdk/build_chicoma_hackathon.sh +++ b/examples/gpmdk/build_chicoma_hackathon.sh @@ -2,8 +2,8 @@ WORKDIR=/usr/projects/icapt/mewall/packages/gpmd rm -rf build_hackathon mkdir build_hackathon cd build_hackathon -cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes" \ --DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero" \ +cmake -DCMAKE_Fortran_COMPILER="ftn" -DPROGRESS_MPI="yes -DLIB="no" -DGPMDK_NVTX="yes"" \ +-DEXTRA_FCFLAGS="-g -O2 -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -lnvToolsExt" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_hackathon/;$WORKDIR/qmd-progress/bml/install_hackathon;$WORKDIR/metis-5.1.0/" ../src/ #-DEXTRA_FCFLAGS="-g -O2 -DHPCTOOLKIT_PROFILE -Wall -Wunused -fopenmp -ffree-line-length-none -ffpe-trap=invalid,overflow,zero -L$HPCTOOLKIT/lib/hpctoolkit -lhpctoolkit" \ make diff --git a/examples/gpmdk/run/TrpCage/sbatch.sh b/examples/gpmdk/run/TrpCage/sbatch.sh index ce6b3672..8b1765bc 100755 --- a/examples/gpmdk/run/TrpCage/sbatch.sh +++ b/examples/gpmdk/run/TrpCage/sbatch.sh @@ -1,19 +1,23 @@ #!/bin/bash -##SBATCH -p gpu +#SBATCH -p gpu ##SBATCH -p gpu_debug ##SBATCH --reservation=gpu_debug ##SBATCH --qos=debug -#SBATCH --time 2:00:00 -#SBATCH -N 64 -##SBATCH -A ichelp_g +#SBATCH --time 0:15:00 +#SBATCH -N 16 +#SBATCH -A ichelp_g ##SBATCH -A w23_macroqmd_g cd ${PWD} -source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh +source /usr/projects/icapt/mewall/packages/gpmd/gpmd/setenv_gpu.sh #export MPICH_ALLREDUCE_NO_SMP=1 export MPICH_SMP_SINGLE_COPY_MODE=NONE #export MPICH_OPT_THREAD_SYNC=0 +<<<<<<< Updated upstream OMP_NUM_THREADS=72 srun -n 64 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh +======= +OMP_NUM_THREADS=32 srun -n 64 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh +>>>>>>> Stashed changes diff --git a/examples/gpmdk/run/TrpCage/wrapper.sh b/examples/gpmdk/run/TrpCage/wrapper.sh index 66b07acf..8377db0d 100644 --- a/examples/gpmdk/run/TrpCage/wrapper.sh +++ b/examples/gpmdk/run/TrpCage/wrapper.sh @@ -3,7 +3,7 @@ export CUDA_VISIBLE_DEVICES=$(echo "${SLURM_LOCALID}%4" |bc) #RUN="nsys profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_gpu/gpmdcov" #RUN="map --profile /usr/projects/icapt/mewall/packages/gpmd/gpmd/build_forge/gpmdcov" -RUN="/usr/projects/icapt/mewall/venado/packages/qmd-progress/examples/gpmdk/build_hackathon/gpmdk" +RUN="/usr/projects/icapt/mewall/packages/gpmd/qmd-progress/examples/gpmdk/build_hackathon/gpmdk" #stdbuf -o0 $RUN input_ICH.in |& tee out_ICH if [ $SLURM_PROCID -eq 0 ]; then stdbuf -o0 $RUN input.in >& out_0 diff --git a/examples/gpmdk/run/water/input.in b/examples/gpmdk/run/water/input.in index dc41c396..7d11e51c 100644 --- a/examples/gpmdk/run/water/input.in +++ b/examples/gpmdk/run/water/input.in @@ -38,8 +38,8 @@ Latte{ #CoordsFile= coords.ltt #CoordsFile= coords_300New.dat #CoordsFile= coords_300_sort.dat - CoordsFile= coords_300.dat - #CoordsFile= coords_2088.dat + #CoordsFile= coords_300.dat + CoordsFile= coords_2088.dat #CoordsFile= "./polyaniline.pdb" #CoordsFile= graphite2048.pdb #CoordsFile= carbon_2197.pdb @@ -99,7 +99,7 @@ GSP2{ BMLType= Ellpack #GraphElement= Orbital GraphElement= Atom - PartitionType= Block + #PartitionType= Block #NodesPerPart= 333 #NodesPerPart= 18 #NodesPerPart= 27 @@ -108,7 +108,7 @@ GSP2{ #NodesPerPart= 48 #NodesPerPart= 150 #NodesPerPart= 1331 - #PartitionType= METIS + PartitionType= METIS #PartitionType= METIS+SA #PartitionType= METIS+KL #PartitionRefinement= None @@ -121,8 +121,8 @@ GSP2{ #PartitionCount= 256 #PartitionCount= 512 #PartitionCount= 16 - PartitionCount= 1 - #PartitionCount= 8 + #PartitionCount= 2 + PartitionCount= 8 #PartitionCount= 1024 #PartitionCount= 32 #PartitionCount= 3200 diff --git a/examples/gpmdk/run/water/sbatch.sh b/examples/gpmdk/run/water/sbatch.sh index ca5ee53c..502b2772 100755 --- a/examples/gpmdk/run/water/sbatch.sh +++ b/examples/gpmdk/run/water/sbatch.sh @@ -9,11 +9,11 @@ ##SBATCH -A w23_macroqmd_g cd ${PWD} -source /usr/projects/icapt/mewall/venado/packages/qmd-progress/scripts/setenv_venado.sh +source /usr/projects/icapt/mewall/packages/gpmd/gpmd/setenv_gpu.sh #export MPICH_ALLREDUCE_NO_SMP=1 export MPICH_SMP_SINGLE_COPY_MODE=NONE #export MPICH_OPT_THREAD_SYNC=0 -OMP_NUM_THREADS=72 srun -n 16 --ntasks-per-node=4 --cpus-per-task=72 bash wrapper.sh +OMP_NUM_THREADS=32 srun -n 8 --ntasks-per-node=4 --cpus-per-task=32 bash wrapper.sh diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index 1ed1e572..fe245028 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -13,6 +13,9 @@ subroutine gpmdcov_Part(ipreMD) integer, allocatable :: graph_h(:,:) integer, allocatable :: graph_p(:,:) + integer, allocatable, save :: graph_p_old(:,:) + integer, allocatable, save :: chindex_old(:,:) + logical :: chindex_same real(dp) :: mls_ii real, allocatable :: onesMat(:,:) integer :: iipt,ipreMD @@ -29,6 +32,11 @@ subroutine gpmdcov_Part(ipreMD) myMdim = gsp2%mdim endif + if(.not.allocated(graph_p_old))then + allocate(graph_p_old(myDim,sy%nats)) + allocate(chindex_old(gpat%totalNodes2,gpat%totalParts)) + endif + if(ipreMD == 1)then call gpmdcov_msMem("gpmdcov_Part", "Before prg_get_covgraph",lt%verbose,myRank) call prg_get_covgraph(sy,nl%nnStruct,nl%nrnnstruct& @@ -45,6 +53,7 @@ subroutine gpmdcov_Part(ipreMD) call prg_get_covgraph_h(sy,nl%nnStruct,nl%nrnnstruct,gsp2%nlgcut,graph_h,myMdim,lt%verbose) call gpmdcov_msII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) + chindex_same = .true. #ifdef DO_MPI !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) do iipt=1,partsInEachRank(myRank) @@ -55,7 +64,12 @@ subroutine gpmdcov_Part(ipreMD) call prg_collect_graph_p(syprt(ipt)%estr%orho,gpat%sgraph(ipt)%llsize,sy%nats,syprt(ipt)%estr%hindex,& gpat%sgraph(ipt)%core_halo_index,graph_p,gsp2%gthreshold,myMdim,lt%verbose) - + +#ifdef DO_MPI + write(*,*)"DEBUG: array sizes are ",size(gpat%sgraph(ipt)%core_halo_index),size(chindex_old) + chindex_same = chindex_same.and.all(gpat%sgraph(ipt)%core_halo_index.eq.chindex_old(:,ipt)) +#endif + call bml_deallocate(syprt(ipt)%estr%orho) enddo @@ -66,8 +80,19 @@ subroutine gpmdcov_Part(ipreMD) #ifdef DO_MPI if (getNRanks() > 1) then - call prg_barrierParallel - call prg_sumIntReduceN(graph_p, myMdim*sy%nats) + call prg_barrierParallel + if(.not.chindex_same)then + call prg_sumIntReduceN(graph_p, myMdim*sy%nats) + write(*,*)"DEBUG: Saving graph details on rank ",myRank,"..." + graph_p_old = graph_p + do iipt=1,gpat%TotalParts + chindex_old(:,iipt) = gpat%sgraph(iipt)%core_halo_index + enddo + write(*,*)"DEBUG: Saved graph details" + else + call prg_sumIntReduceN(graph_p, myMdim*sy%nats) + write(*,*)"DEBUG: Not saving graph details" + endif ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) endif #endif From 77f9065f435f624a0bf3ddbf3c9b5c7d77e1ebc5 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 7 Aug 2024 11:31:00 -0600 Subject: [PATCH 31/69] Beginning update of graph MPI --- examples/gpmdk/run/water/input_single.in | 8 +++++--- examples/gpmdk/run/water/run.sh | 3 ++- examples/gpmdk/run/water/run_single.sh | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/gpmdk/run/water/input_single.in b/examples/gpmdk/run/water/input_single.in index 967142af..fc5cd443 100644 --- a/examples/gpmdk/run/water/input_single.in +++ b/examples/gpmdk/run/water/input_single.in @@ -31,14 +31,15 @@ Latte{ TimeRatio= 10.0 TimeStep= 0.2 #TimeStep= 0.00 - MDSteps= 10000 + MDSteps= 10 #ParamPath= "../sulfurTBparam" - ParamPath= "../tests/latteTBparams" + ParamPath= "../../tests/latteTBparams" #ParamPath= "../latteTBparams_orig" #CoordsFile= coords.ltt #CoordsFile= coords_300New.dat #CoordsFile= coords_300_sort.dat CoordsFile= coords_300.dat + #CoordsFile= coords_891.pdb #CoordsFile= coords_2088.dat #CoordsFile= "./polyaniline.pdb" #CoordsFile= graphite2048.pdb @@ -176,7 +177,8 @@ GPMD{ WriteTrajectory= T WriteCoordsEach= 10 LangevinMethod= Siva - LangevinDynamics= T + LangevinDynamics= F LangevinGamma= 0.01 InitialTemperature= 300.0 + UseVectSKBlock= T } diff --git a/examples/gpmdk/run/water/run.sh b/examples/gpmdk/run/water/run.sh index 0696145d..cf8d45ca 100755 --- a/examples/gpmdk/run/water/run.sh +++ b/examples/gpmdk/run/water/run.sh @@ -7,7 +7,8 @@ #OMP_NUM_THREADS=36 mpiexec -np 1 $RUN input.in | tee out #OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out -OMP_NUM_THREADS=32 srun -n 4 -c 32 bash wrapper.sh +#OMP_NUM_THREADS=32 srun -n 4 -c 32 bash wrapper.sh +OMP_NUM_THREADS=64 srun -n 1 -c 64 bash wrapper.sh #OMP_NUM_THREADS=64 $RUN input.in | tee out #OMP_NUM_THREADS=36 $RUN input.in | tee out #$RUN input.in | tee out diff --git a/examples/gpmdk/run/water/run_single.sh b/examples/gpmdk/run/water/run_single.sh index 5b5bca02..96be51f0 100755 --- a/examples/gpmdk/run/water/run_single.sh +++ b/examples/gpmdk/run/water/run_single.sh @@ -8,7 +8,7 @@ RUN="$PWD/../../build_hackathon/gpmdk" #OMP_NUM_THREADS=36 mpirun -np 3 --map-by socket:pe=18 --bind-to core:overload-allowed $RUN input.in | tee out #OMP_NUM_THREADS=64 $RUN input.in >& out -OMP_NUM_THREADS=72 srun -n 1 -c 72 $RUN input.in >& out +OMP_NUM_THREADS=64 srun -n 1 -c 64 $RUN input_single.in >& out #OMP_NUM_THREADS=36 $RUN input.in | tee out #$RUN input.in | tee out From ee062630cb4e4c7c1ee3ec5c410fe49d7d861968 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Thu, 8 Aug 2024 08:20:13 -0600 Subject: [PATCH 32/69] Preliminary low-communication graph update introduced. Not tested. --- examples/gpmdk/src/gpmdcov_part.F90 | 242 +++++++++++++++++++++++++--- 1 file changed, 219 insertions(+), 23 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index fe245028..4c081a71 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -10,12 +10,14 @@ subroutine gpmdcov_Part(ipreMD) use gpmdcov_reshuffle_mod use gpmdcov_writeout_mod use gpmdcov_allocation_mod - + implicit none integer, allocatable :: graph_h(:,:) integer, allocatable :: graph_p(:,:) integer, allocatable, save :: graph_p_old(:,:) - integer, allocatable, save :: chindex_old(:,:) - logical :: chindex_same + integer, allocatable, save :: G_added(:,:), G_removed(:,:), G_updated(:,:) + integer, allocatable, save :: N_added(:), N_removed(:), NNZ1(:), NNZ2(:), NNZ_updated(:) + logical, allocatable, save :: v(:) + integer :: na, usz, k, ktot real(dp) :: mls_ii real, allocatable :: onesMat(:,:) integer :: iipt,ipreMD @@ -31,11 +33,6 @@ subroutine gpmdcov_Part(ipreMD) else myMdim = gsp2%mdim endif - - if(.not.allocated(graph_p_old))then - allocate(graph_p_old(myDim,sy%nats)) - allocate(chindex_old(gpat%totalNodes2,gpat%totalParts)) - endif if(ipreMD == 1)then call gpmdcov_msMem("gpmdcov_Part", "Before prg_get_covgraph",lt%verbose,myRank) @@ -43,7 +40,23 @@ subroutine gpmdcov_Part(ipreMD) ,gsp2%bml_type,gsp2%covgfact,g_bml,myMdim,lt%verbose) call gpmdcov_msMem("gpmdcov_Part", "After prg_get_covgraph",lt%verbose,myRank) else - +#ifdef DO_MPI + na = sy%nats + usz = 100 + if(.not.allocated(graph_p_old))then + allocate(graph_p_old(myMdim,na)) + graph_p_old = 0 + allocate(v(na)) + allocate(G_added(usz,na)) + allocate(G_removed(usz,na)) + allocate(G_updated(usz,na)) + allocate(N_added(na)) + allocate(N_removed(na)) + allocate(NNZ1(na)) + allocate(NNZ2(na)) + allocate(NNZ_updated(na)) + endif +#endif !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !Anders' way of graph construction. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -53,7 +66,6 @@ subroutine gpmdcov_Part(ipreMD) call prg_get_covgraph_h(sy,nl%nnStruct,nl%nrnnstruct,gsp2%nlgcut,graph_h,myMdim,lt%verbose) call gpmdcov_msII("gpmdcov_Part","In prg_get_covgraph_h ..."//to_string(mls()-mls_ii)//" ms",lt%verbose,myRank) - chindex_same = .true. #ifdef DO_MPI !do ipt= gpat%localPartMin(myRank), gpat%localPartMax(myRank) do iipt=1,partsInEachRank(myRank) @@ -65,11 +77,6 @@ subroutine gpmdcov_Part(ipreMD) call prg_collect_graph_p(syprt(ipt)%estr%orho,gpat%sgraph(ipt)%llsize,sy%nats,syprt(ipt)%estr%hindex,& gpat%sgraph(ipt)%core_halo_index,graph_p,gsp2%gthreshold,myMdim,lt%verbose) -#ifdef DO_MPI - write(*,*)"DEBUG: array sizes are ",size(gpat%sgraph(ipt)%core_halo_index),size(chindex_old) - chindex_same = chindex_same.and.all(gpat%sgraph(ipt)%core_halo_index.eq.chindex_old(:,ipt)) -#endif - call bml_deallocate(syprt(ipt)%estr%orho) enddo @@ -81,19 +88,208 @@ subroutine gpmdcov_Part(ipreMD) #ifdef DO_MPI if (getNRanks() > 1) then call prg_barrierParallel - if(.not.chindex_same)then + if((gsp2%parteach == 1) .or. (mod(mdstep,gsp2%parteach)==1) .or. (mdstep <= 1))then call prg_sumIntReduceN(graph_p, myMdim*sy%nats) - write(*,*)"DEBUG: Saving graph details on rank ",myRank,"..." + write(*,*)"DEBUG: Doing full graph reduction at mdstep ",mdstep graph_p_old = graph_p - do iipt=1,gpat%TotalParts - chindex_old(:,iipt) = gpat%sgraph(iipt)%core_halo_index - enddo - write(*,*)"DEBUG: Saved graph details" else + !MATLAB code + ! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! %% Detects new and removed edges between two graphs G1 and G2 %% + ! %% The routine does not require ordering %% + ! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! clear; + ! N = 10; M = 5; % N nodes and max M edges + + ! % Create first graph randomly + ! G1 = zeros(N,M); + ! NNZ1 = floor(1 + M*rand(10,1)); % [1,M] for each node + ! TIO = [1:N]'; + ! for i = 1:N + ! for k = 1:7 % reshuffle ordering + ! a = floor(1 + N*rand(1)); + ! b = floor(1 + N*rand(1)); + ! tmp = TIO(a); TIO(a) = TIO(b); TIO(b) = tmp; + ! end + ! for j = 1:NNZ1(i) + ! G1(i,j) = TIO(j); + ! end + ! end + + ! % Create second graph randomly + ! G2 = zeros(N,M); + ! NNZ2 = floor(1 + M*rand(10,1)); % [1,M] for each node + ! TIO = [1:N]'; + ! for i = 1:N + ! for k = 1:7 % reshuffle ordering + ! a = floor(1 + N*rand(1)); + ! b = floor(1 + N*rand(1)); + ! tmp = TIO(a); TIO(a) = TIO(b); TIO(b) = tmp; + ! end + ! for j = 1:NNZ2(i) + ! G2(i,j) = TIO(j); + ! end + ! end + + ! % Analyze the difference from G1 to G2 + + ! % Added edges + ! G_added = zeros(N,M); N_added = zeros(N,1); + ! v = zeros(1,N); + ! for i = 1:N + ! for j = 1:NNZ1(i) + ! v(G1(i,j)) = 1; + ! end + ! k = 0; + ! for j = 1:NNZ2(i) + ! if (v(G2(i,j)) == 0) + ! k = k + 1; + ! G_added(i,k) = G2(i,j); + ! end + ! end + ! N_added(i) = k; % Number of added edges for each vertex i + ! v(G1(i,1:NNZ1(i))) = 0; + ! v(G2(i,1:NNZ2(i))) = 0; + ! end + + ! % Removed edges + ! G_removed = zeros(N,M); N_removed = zeros(N,1); + ! v = zeros(1,N); + ! for i = 1:N + ! for j = 1:NNZ2(i) + ! v(G2(i,j)) = 1; + ! end + ! k = 0; + ! for j = 1:NNZ1(i) + ! if (v(G1(i,j)) == 0) + ! k = k + 1; + ! G_removed(i,k) = G1(i,j); + ! end + ! end + ! N_removed(i) = k; % Number of removed edges for each vertex i + ! v(G1(i,1:NNZ1(i))) = 0; + ! v(G2(i,1:NNZ2(i))) = 0; + ! end + + ! %% Use G_removed and G_added to update from G1 to G2 + ! G_Updated = zeros(N,M); + ! NNZ_Updated = zeros(N,1); + ! v = zeros(N,1); % Temporary vector that keeps track of elements that are there and then removed. + ! for i = 1:N + ! for j = 1:NNZ1(i) + ! v(G1(i,j)) = 1; + ! end + ! for j = 1:N_removed(i) + ! v(G_removed(i,j)) = 0; % Remove edges + ! end + ! cnt = 0; + ! for j = 1:NNZ1(i) + ! if v(G1(i,j)) > 0; % Account only for the remaining edges + ! cnt = cnt + 1; + ! G_Updated(i,cnt) = G1(i,j); + ! end + ! NNZ_Updated(i) = cnt + N_added(i); + ! end + ! for j = cnt+1:NNZ_Updated(i) + ! G_Updated(i,j) = G_added(i,j-cnt); % Add new edges at the end + ! end + ! end + + ! % Check NNZ_Updated: NNZ_Updated = NNZ1 + N_Added - N_Removed + ! [NNZ_Updated, NNZ1 + N_added - N_removed] + + ! % Check NNZ_Updated: G_Updated = G2 But edges are not in the same order + + ! % Added edges + ktot = 0 + NNZ1 = count(graph_p_old.ne.0,DIM=1) + NNZ2 = count(graph_p.ne.0,DIM=1) + + G_added = 0 + N_added = 0 + v = .false. + do iipt=1,partsInEachRank(myRank) + ipt= reshuffle(iipt,myRank) + do ii = 1,gpat%sgraph(ipt)%llsize + i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + do j = 1,NNZ1(i) + v(graph_p_old(j,i)) = .true. + end do + k = 0; + do j = 1,NNZ2(i) + if (v(graph_p(j,i)) .eqv. .false.)then + k = k + 1 + G_added(k,i) = graph_p(j,i) + endif + end do + N_added(i) = k ! Number of added edges for each vertex i + ktot = ktot + k + v(graph_p_old(1:NNZ1(i),i)) = .false. + v(graph_p(1:NNZ2(i),i)) = .false. + end do + enddo + ! Removed edges + G_removed = 0 + N_removed = 0 + v = .false. + do iipt=1,partsInEachRank(myRank) + ipt= reshuffle(iipt,myRank) + do ii = 1,gpat%sgraph(ipt)%llsize + i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + do j = 1,NNZ2(i) + v(graph_p(j,i)) = .true. + end do + k = 0; + do j = 1,NNZ1(i) + if (v(graph_p_old(j,i)) .eqv. .false.)then + k = k + 1 + G_removed(k,i) = graph_p_old(j,i) + endif + end do + N_removed(i) = k ! Number of added edges for each vertex i + ktot = ktot + k + v(graph_p_old(1:NNZ1(i),i)) = .false. + v(graph_p(1:NNZ2(i),i)) = .false. + end do + enddo + write(*,*)"DEBUG: ktot = ",ktot call prg_sumIntReduceN(graph_p, myMdim*sy%nats) - write(*,*)"DEBUG: Not saving graph details" + write(*,*)"DEBUG: Doing graph update reduction at mdstep ",mdstep + ! %% Use G_removed and G_added to update from G1 to G2 + G_updated = 0 + NNZ_updated = 0 + v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. + do iipt=1,partsInEachRank(myRank) + ipt= reshuffle(iipt,myRank) + do ii = 1,gpat%sgraph(ipt)%llsize + i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + do j = 1,NNZ1(i) + v(graph_p_old(j,i)) = .true. + end do + do j = 1,N_removed(i) + v(G_removed(j,i)) = .false. ! % Remove edges + end do + k = 0 + do j = 1,NNZ1(i) + if (v(graph_p_old(j,i)) .eqv. .true.)then ! % Account only for the remaining edges + k = k + 1; + G_updated(k,i) = graph_p_old(j,i); + end if + NNZ_updated(i) = k + N_added(i); + end do + do j = k+1,NNZ_updated(i) + G_updated(j,i) = G_added(j-k,i) ! Add new edges at the end + end do + end do + enddo + ! % Check NNZ_Updated: NNZ_Updated = NNZ1 + N_Added - N_Removed + ! [NNZ_Updated, NNZ1 + N_added - N_removed] + + ! % Check NNZ_Updated: G_Updated = G2 But edges are not in the same order + + graph_p_old = graph_p endif - ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) + ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) endif #endif From df076234c1fbcbe8b69d1092f43fe792f0a6bdb3 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Thu, 8 Aug 2024 11:23:50 -0600 Subject: [PATCH 33/69] Working subgraph graph update method with less MPI communication --- examples/gpmdk/src/gpmdcov_part.F90 | 70 ++++++++++++++++++----------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index 4c081a71..8b9eeafd 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -16,8 +16,8 @@ subroutine gpmdcov_Part(ipreMD) integer, allocatable, save :: graph_p_old(:,:) integer, allocatable, save :: G_added(:,:), G_removed(:,:), G_updated(:,:) integer, allocatable, save :: N_added(:), N_removed(:), NNZ1(:), NNZ2(:), NNZ_updated(:) - logical, allocatable, save :: v(:) - integer :: na, usz, k, ktot + logical, allocatable, save :: v(:), v_check(:), check_graph + integer :: n_atoms, max_updates, k, ktot real(dp) :: mls_ii real, allocatable :: onesMat(:,:) integer :: iipt,ipreMD @@ -41,20 +41,21 @@ subroutine gpmdcov_Part(ipreMD) call gpmdcov_msMem("gpmdcov_Part", "After prg_get_covgraph",lt%verbose,myRank) else #ifdef DO_MPI - na = sy%nats - usz = 100 + n_atoms = sy%nats + max_updates = 100 if(.not.allocated(graph_p_old))then - allocate(graph_p_old(myMdim,na)) + allocate(graph_p_old(myMdim,n_atoms)) graph_p_old = 0 - allocate(v(na)) - allocate(G_added(usz,na)) - allocate(G_removed(usz,na)) - allocate(G_updated(usz,na)) - allocate(N_added(na)) - allocate(N_removed(na)) - allocate(NNZ1(na)) - allocate(NNZ2(na)) - allocate(NNZ_updated(na)) + allocate(v(n_atoms)) + allocate(v_check(n_atoms)) + allocate(G_added(max_updates,n_atoms)) + allocate(G_removed(max_updates,n_atoms)) + allocate(G_updated(max_updates,n_atoms)) + allocate(N_added(n_atoms)) + allocate(N_removed(n_atoms)) + allocate(NNZ1(n_atoms)) + allocate(NNZ2(n_atoms)) + allocate(NNZ_updated(n_atoms)) endif #endif !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -252,17 +253,25 @@ subroutine gpmdcov_Part(ipreMD) v(graph_p(1:NNZ2(i),i)) = .false. end do enddo + ! % Check NNZ_Updated: NNZ_Updated = NNZ1 + N_Added - N_Removed + ! [NNZ_Updated, NNZ1 + N_added - N_removed] + + ! % Check NNZ_Updated: G_Updated = G2 But edges are not in the same order + write(*,*)"DEBUG: ktot = ",ktot call prg_sumIntReduceN(graph_p, myMdim*sy%nats) write(*,*)"DEBUG: Doing graph update reduction at mdstep ",mdstep - ! %% Use G_removed and G_added to update from G1 to G2 + ! %% Use G_removed and G_added to update from G1 to G2 + call prg_sumIntReduceN(G_added,n_atoms*max_updates) + call prg_sumIntReduceN(G_removed,n_atoms*max_updates) G_updated = 0 NNZ_updated = 0 v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. - do iipt=1,partsInEachRank(myRank) - ipt= reshuffle(iipt,myRank) - do ii = 1,gpat%sgraph(ipt)%llsize - i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + ! do iipt=1,partsInEachRank(myRank) + ! ipt= reshuffle(iipt,myRank) + ! do ii = 1,gpat%sgraph(ipt)%llsize + ! i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + do i = 1,n_atoms do j = 1,NNZ1(i) v(graph_p_old(j,i)) = .true. end do @@ -280,13 +289,24 @@ subroutine gpmdcov_Part(ipreMD) do j = k+1,NNZ_updated(i) G_updated(j,i) = G_added(j-k,i) ! Add new edges at the end end do + check_graph = .true. + if (check_graph)then + v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. + v_check = .false. + if (NNZ_updated(i) == NNZ2(i))then + write(*,*)"DEBUG: Number of nonzero elements is the same" + end if + do j = 1,NNZ2(i) + v(graph_p(j,i)) = .true. + v_check(G_updated(j,i)) = .true. + end do + if (all(v.eqv.v_check))then + write(*,*)"DEBUG: elements are the same" + end if + end if end do - enddo - ! % Check NNZ_Updated: NNZ_Updated = NNZ1 + N_Added - N_Removed - ! [NNZ_Updated, NNZ1 + N_added - N_removed] - - ! % Check NNZ_Updated: G_Updated = G2 But edges are not in the same order - + !end do + graph_p = G_updated graph_p_old = graph_p endif ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) From a4a61d98d92d541432d64079d3e1e911b96deb9b Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 8 Aug 2024 15:00:37 -0600 Subject: [PATCH 34/69] Added new script --- examples/gpmdk/run_lib/main_bch_dspl.py | 95 ++++++++++++++++++++++++ examples/gpmdk/src/gpmd.py | 28 +++---- examples/gpmdk/src/gpmdcov_highorder.F90 | 50 ++++++------- examples/gpmdk/src/gpmdcov_lib.F90 | 26 ++----- examples/gpmdk/src/gpmdcov_lib_mod.F90 | 33 ++++---- 5 files changed, 149 insertions(+), 83 deletions(-) create mode 100755 examples/gpmdk/run_lib/main_bch_dspl.py diff --git a/examples/gpmdk/run_lib/main_bch_dspl.py b/examples/gpmdk/run_lib/main_bch_dspl.py new file mode 100755 index 00000000..c9cd52cd --- /dev/null +++ b/examples/gpmdk/run_lib/main_bch_dspl.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import numpy as np +from sdc_system import * +from sdc_ptable import ptable +import ctypes as ct +import os +import scipy.linalg as sp +from gpmd import * + + +def get_displ_born_charges(latticeVectors,symbols,atomTypes,coordsIn,verb): + + nats = len(atomTypes) + bornDispl = np.zeros((nats,3,3)) + dspl = 1.0E-5 + field = np.zeros((3)) + coords = np.copy(coordsIn) + err,charges_out,forces_out,dipole_saved, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + for j in range(nats): + coords[j,0] = coordsIn[j,0] + dspl + err,charges_out,forces_out,dipole_out_p, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,0] = coordsIn[j,0] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + bornDispl[j,:,0] = (dipole_out_p - dipole_out_m)/(2*dspl) + coords[j,0] = coordsIn[j,0] + dspl + + for j in range(nats): + coords[j,1] = coordsIn[j,1] + dspl + err,charges_out,forces_out,dipole_out_p,energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,1] = coordsIn[j,1] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,1] = coordsIn[j,1] + dspl + bornDispl[j,:,1] = (dipole_out_p - dipole_out_m)/(2*dspl) + for j in range(nats): + coords[j,2] = coordsIn[j,2] + dspl + err,charges_out,forces_out,dipole_out_p, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,2] = coordsIn[j,2] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,2] = coordsIn[j,2] + dspl + bornDispl[j,:,2] = (dipole_out_p - dipole_out_m)/(2*dspl) + + + bchMat = np.zeros((3,3)) + bornCharges = np.zeros((nats)) + for i in range(len(bornDispl[:,0,0])): + bchMat[:,:] = bornDispl[i,:,:] + E,Q = sp.eigh(bchMat) + bornCharges[i] = sum(E)/3.0 + + return bornCharges, bornDispl + + + +############################### +## Main starts here +############################### + +#****************************** +#Check forces with no Field +#****************************** +verb = 0 +#Load the coordinates and atom types +latticeVectors,symbols,atomTypes,coords0 = read_pdb_file("H2O.pdb",lib="None",verb=True) + +nats = len(atomTypes) +dspl = 0.00001 +field = np.zeros((3)) +field[0] = 0.0 + +#Get forces at coords0 +err,charges_out,forces0,dipole_out, energyp = gpmd(latticeVectors,symbols,atomTypes,coords0,field,verb) + +coordsP = np.copy(coords0) +coordsP[0,0] = coords0[0,0] + dspl +err,charges_out_pl,forces_out_pl,dipole_out, energyp = gpmd(latticeVectors,symbols,atomTypes,coordsP,field,verb) + +coordsM = np.copy(coords0) +coordsM[0,0] = coords0[0,0] - dspl +err,charges_out_pl,forces_out_pl,dipole_out, energym = gpmd(latticeVectors,symbols,atomTypes,coordsM,field,verb) + +print("Check: dE/dx,F",(energyp - energym)/(2*dspl),forces0[0,0]) +exit(0) +#****************************** +#Compute Born Charges using displacements +#****************************** + +bornChDiag,bornChargesDspl = get_displ_born_charges(latticeVectors,symbols,atomTypes,coords0,verb) + +for j in range(len(atomTypes)): + print("Atom index",j,"","Mulliquen Charges:",charges_out[j]) + print("Born Charges:\n",bornChargesDspl[j,:,:]) + + +exit(0) diff --git a/examples/gpmdk/src/gpmd.py b/examples/gpmdk/src/gpmd.py index 85171b35..629cc35c 100755 --- a/examples/gpmdk/src/gpmd.py +++ b/examples/gpmdk/src/gpmd.py @@ -14,6 +14,7 @@ # @param symbols Symbols for each atom type, e.g., the element symbol of the first atom is symbols[types[0]] # @param atomTypes Type for each atom, e.g., the first atom is of type "types[0]" # @param coords for each atom, e.g., z-coordinate of the frist atom is coords[0,2] +# @param field Applied field, e.g., field[0] = 1.0 (setting x-direction of the field to 1.0) # @param verb Verbosity level for gpmd output # def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): @@ -44,6 +45,7 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): chargesFlat_out = np.zeros(nats) #We call this one Flat just for consistency dipoleFlat_out = np.zeros(3) #Same here fieldFlat_in = np.zeros(3) #Same here + energyFlat_out = np.zeros(1) #bornchFlat_out = np.zeros(9*nats) for i in range(nats): @@ -69,7 +71,8 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): #Specify arguments as a pointers to pass to Fortran f.argtypes=[ct.c_int,ct.c_int,ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),\ ct.POINTER(ct.c_int),ct.POINTER(ct.c_int),ct.POINTER(ct.c_double),\ - ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),ct.c_int] + ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),ct.POINTER(ct.c_double),\ + ct.POINTER(ct.c_double),ct.c_int] #Inputs coords_ptr = coordsFlat_in.ctypes.data_as(ct.POINTER(ct.c_double)) @@ -82,18 +85,22 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): charges_ptr = chargesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) forces_ptr = forcesFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) dipole_ptr = dipoleFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + energy_ptr = energyFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) + #energy_out = 0.0 #bornch_ptr = bornchFlat_out.ctypes.data_as(ct.POINTER(ct.c_double)) #Call to the fortran funtion err = f(ct.c_int(nats),ct.c_int(nTypes),coords_ptr,latticeVectors_ptr,\ atomTypes_ptr,atomicNumbers_ptr,field_ptr,charges_ptr,forces_ptr,\ - dipole_ptr,ct.c_int(verb)) + dipole_ptr,energy_ptr,ct.c_int(verb)) charges_out = np.zeros(nats) charges_out[:] = chargesFlat_out[:] dipole_out = np.zeros(3) dipole_out[:] = dipoleFlat_out[:] - + energy_out = energyFlat_out[0] + print("energy_out_python",energy_out) + #Back to a 2D array for the forces forces_out = np.zeros((nats,3)) #bornch_out = np.zeros((nats,9)) @@ -102,20 +109,7 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): forces_out[i,1] = forcesFlat_out[i*3 + 1] forces_out[i,2] = forcesFlat_out[i*3 + 2] - # bornch_out[i,0] = bornchFlat_out[i*9 + 0] - # bornch_out[i,1] = bornchFlat_out[i*9 + 1] - # bornch_out[i,2] = bornchFlat_out[i*9 + 2] - # - # bornch_out[i,3] = bornchFlat_out[i*9 + 3] - # bornch_out[i,4] = bornchFlat_out[i*9 + 4] - # bornch_out[i,5] = bornchFlat_out[i*9 + 5] - # - # bornch_out[i,6] = bornchFlat_out[i*9 + 6] - # bornch_out[i,7] = bornchFlat_out[i*9 + 7] - # bornch_out[i,8] = bornchFlat_out[i*9 + 8] - - - return err, charges_out, forces_out, dipole_out + return err, charges_out, forces_out, dipole_out, energy_out diff --git a/examples/gpmdk/src/gpmdcov_highorder.F90 b/examples/gpmdk/src/gpmdcov_highorder.F90 index 696b96ed..54298c47 100644 --- a/examples/gpmdk/src/gpmdcov_highorder.F90 +++ b/examples/gpmdk/src/gpmdcov_highorder.F90 @@ -19,12 +19,12 @@ module gpmdcov_highorder_mod use prg_response_mod - public :: gpmdcov_get_born_charges_v1 - public :: gpmdcov_get_born_charges_v2 + ! public :: gpmdcov_get_born_charges_v1 + !public :: gpmdcov_get_born_charges_v2 public :: gpmdcov_field_forces contains - +#ifdef FULL !> Computes the Born charges !! \brief Based on the derivative of the dipole moment with respect !! to the coordinates. @@ -245,51 +245,49 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) deallocate(forcesSaved) end subroutine gpmdcov_get_born_charges_v2 - +#endif !> Field forces - subroutine gpmdcov_get_field_forces(norbi,field_in) + subroutine gpmdcov_get_field_forces(norbi,znucs,field_in) implicit none real(dp),allocatable :: dipoleMoment(:),savedCharges(:) real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) real(dp) :: lambda,intensity,field(3),threshold real(dp), allocatable :: forcesSaved(:,:) real(dp), allocatable, intent(in) :: field_in(:) - integer :: atomI, verbose - integer, allocatable, intent(in) :: norbi(:) - type(bml_matrix_t) :: prt_bml - real(dp) :: relperm, keconst, tfact - real(dp) :: intensity0,dintensity + integer :: atomI, verbose, k + integer, allocatable, intent(in) :: norbi(:), znucs(:) + type(bml_matrix_t) :: prt_bml, oham + real(dp) :: relperm, keconst, tfact, kappa + real(dp) :: intensity0,dintensity,KE - lambda = 1.0_dp !2.0_dp/14.3996437701414_dp threshold = 0.0_dp verbose = 3 - !This will assume only one part - call gpmdcov_InitParts() - allocate(forcesSaved(3,sy%nats)) - - intensity0 = 0.0000_dp !eV/Ang - dintensity = 0.001_dp - intensity = 1.0D-4 + kappa = 1.0_dp + write(*,*)field_in field = field_in/norm2(field_in) intensity = norm2(field_in) + lambda = kappa*intensity + write(*,*)(sy%coordinate) call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) - call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) - sy%net_charge = 0.0_dp - call gpmdcov_InitParts() - call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V - call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) + call prg_pert_constant_field(field,1.0_dp,sy%coordinate,1.0_dp& + ,prt_bml,0.0_dp,sy%spindex,norbi,verbose,syprt(1)%estr%over) + call bml_print_matrix("Pert",prt_bml,0,6,0,6) + call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,-lambda,threshold) !H = H + lambda*V + call gpmdcov_Diagonalize_H0() call gpmdcov_muFromParts() call gpmdcov_FirstCharges(eig) - call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) + call gpmdcov_DM_Min_Eig(1000,sy%net_charge,.true.,.false.) call gpmdcov_EnergAndForces(sy%net_charge) + + do k = 1,sy%nats + sy%force(:,k) = sy%force(:,k) + field_in(:)*sy%net_charge(k) + enddo end subroutine gpmdcov_get_field_forces - end module gpmdcov_highorder_mod diff --git a/examples/gpmdk/src/gpmdcov_lib.F90 b/examples/gpmdk/src/gpmdcov_lib.F90 index d0245f11..a25b0033 100644 --- a/examples/gpmdk/src/gpmdcov_lib.F90 +++ b/examples/gpmdk/src/gpmdcov_lib.F90 @@ -26,7 +26,7 @@ !! \param verb_in Verbosity level. !! function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomicNumbers_in,& - &field_in,charges_out,forces_out,dipole_out,verb_in) result(err) bind(c, name='gpmd_compute') + &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) result(err) bind(c, name='gpmd_compute') use iso_c_binding, only: c_char, c_double, c_int, c_bool use gpmdcov_vars @@ -39,8 +39,7 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi real(c_double), intent(inout) :: field_in(3) real(c_double), intent(inout) :: forces_out(3*nats) real(c_double), intent(inout) :: charges_out(nats) - real(c_double), intent(inout) :: dipole_out(3) - !real(c_double), intent(inout) :: bornch_out(9*nats) + real(c_double), intent(inout) :: dipole_out(3), energy_out(1) integer(c_int), intent(inout) :: atomTypes_in(nats) integer(c_int) ,intent(inout) :: atomicNumbers_in(nTypes) real(c_double) ,intent(inout) :: latticeVectors_in(9) @@ -64,7 +63,6 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi allocate(charges(nats)) allocate(forces(3,nats)) allocate(dipole(3)) - !allocate(bornch(9,nats)) allocate(field(3)) !Note that arrays appear in another order. We need to rearange @@ -98,8 +96,9 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi verb = verb_in call gpmd_compute(coords,atomTypes,atomicNumbers,latticeVectors,& - &field,charges,forces,dipole,verb) + &field,charges,forces,dipole,energy_out,verb) + write(*,*)"energy_out",energy_out err = .false. !We vectorize/flatten the forces to send back to python @@ -109,23 +108,9 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi forces_out((k-1)*3 + 3) = forces(3,k) enddo -! do k = 1, nats -! bornch_out((k-1)*9 + 1) = bornch(1,k) -! bornch_out((k-1)*9 + 2) = bornch(2,k) -! bornch_out((k-1)*9 + 3) = bornch(3,k) -! -! bornch_out((k-1)*9 + 4) = bornch(4,k) -! bornch_out((k-1)*9 + 5) = bornch(5,k) -! bornch_out((k-1)*9 + 6) = bornch(6,k) -! -! bornch_out((k-1)*9 + 7) = bornch(7,k) -! bornch_out((k-1)*9 + 8) = bornch(8,k) -! bornch_out((k-1)*9 + 9) = bornch(9,k) -! enddo - charges_out(:) = charges(:) dipole_out(:) = dipole(:) - + deallocate(coords) deallocate(forces) deallocate(charges) @@ -133,7 +118,6 @@ function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomi deallocate(atomTypes) deallocate(atomicNumbers) deallocate(dipole) - !deallocate(bornch) return diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 index 6554e156..6852d4f2 100644 --- a/examples/gpmdk/src/gpmdcov_lib_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -38,7 +38,7 @@ module gpmdcov_lib_mod !subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& ! &charges_out,forces_out,verb_in) subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& - &field_in,charges_out,forces_out,dipole_out,verb_in) + &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) ! Local modules @@ -77,6 +77,7 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors !real(dp), allocatable, intent(inout) :: bornch_out(:,:) real(dp), allocatable, intent(inout) :: dipole_out(:) integer, allocatable, intent(in) :: atomTypes_in(:) + real(dp) :: energy_out(1) integer, intent(in) :: verb_in integer :: k integer, allocatable, intent(in) :: atomic_numbers_in(:) @@ -157,10 +158,12 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors if(lt%stopAt == "gpmdcov_Part") stop !> Initialize partitions. - call gpmdcov_InitParts() - if(lt%stopAt == "gpmdcov_InitParts") stop - !> Comput first charges. + if(lt%method == "DiagEfFull") eig = .false. + + call gpmdcov_InitParts() + + if(norm2(field_in) < 1.0D-10)then if(.not. eig) then call gpmdcov_Diagonalize_H0() if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then @@ -168,8 +171,6 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors endif endif call gpmdcov_FirstCharges(eig) - if(lt%stopAt == "gpmdcov_FirstCharges") return - !> First SCF loop up to maxscf. if(eig)then @@ -177,31 +178,25 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors else call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) endif - - if(lt%stopAt == "gpmdcov_DM_Min") return !> First calculation of energies and forces. call gpmdcov_EnergAndForces(sy%net_charge) - + !stop + + else + call gpmdcov_get_field_forces(tb%norbi,atomic_numbers_in,field_in) + endif + charges_out(:) = sy%net_charge(:) forces_out(:,:) = sy%force(:,:) + energy_out(1) = Epot if(allocated(dipoleMoment)) deallocate(dipoleMoment) allocate(dipoleMoment(3)) factor = 1.0_dp call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) dipole_out(:) = dipoleMoment(:) - write(*,*)"Dipole Moment=",dipoleMoment - !call gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) - if(norm2(field_in) > 1.0D-10)then - call gpmdcov_get_field_forces(tb%norbi,field_in) - forces_out(:,:) = sy%force(:,:) - endif - !call gpmdcov_get_born_charges_v2(dipoleMoment,bornCharges,tb%norbi) - !bornch_out(:,:) = bornCharges(:,:) - !write(*,*)"Born F",bornCharges(:,1) - if(lt%stopAt == "gpmdcov_Energ") then lib_init = .true. call gpmdcov_Finalize() From 6f08dcdc92903e1ecfcb49098c4bfe32897287c7 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 8 Aug 2024 16:01:58 -0600 Subject: [PATCH 35/69] added bch dslp script --- examples/gpmdk/run/TrpCage/input.in | 21 +- examples/gpmdk/run/water/gpmd_input.in | 98 +++++--- examples/gpmdk/run/water/input.in | 9 + .../run_lib/main_field_induced_forces.py | 209 ++++++++++++++++-- examples/gpmdk/src/gpmdcov_energandforces.F90 | 18 +- examples/gpmdk/src/gpmdcov_init.F90 | 6 +- src/prg_graph_mod.F90 | 2 +- src/prg_partition_mod.F90 | 1 + 8 files changed, 303 insertions(+), 61 deletions(-) diff --git a/examples/gpmdk/run/TrpCage/input.in b/examples/gpmdk/run/TrpCage/input.in index 22136780..00fd70ab 100644 --- a/examples/gpmdk/run/TrpCage/input.in +++ b/examples/gpmdk/run/TrpCage/input.in @@ -14,7 +14,7 @@ Latte{ MDim= -1 #Threshold= 1.0d-5 Threshold= 0.0 - Verbose= 2 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + Verbose= 5 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) #SCF variables# #StopAt= "gpmdcov_Energ" #StopAt= "gpmdcov_DM_Min" @@ -75,8 +75,13 @@ Latte{ #CoordsFile= minimized_TrpCage.dat #CoordsFile= Mac1_latte.dat #CoordsFile= ICH.dat +<<<<<<< Updated upstream CoordsFile= input.dat NlistEach= 10 +======= + CoordsFile= input1.dat + NlistEach= 1 +>>>>>>> Stashed changes #BMLDistributionType= Distributed MuCalcType= FromParts #MuCalcType= Combined @@ -98,7 +103,6 @@ SP2{ #Graph-based SP2 parameters GSP2{ - BMLType= Ellpack #GraphElement= Orbital GraphElement= Atom @@ -111,7 +115,8 @@ GSP2{ #NodesPerPart= 48 #NodesPerPart= 150 #NodesPerPart= 1331 - PartitionType= METIS + PartitionType= Sedacs + #PartitionType= METIS #PartitionType= METIS+SA #PartitionType= METIS+KL #PartitionRefinement= None @@ -120,6 +125,7 @@ GSP2{ NLGraphCut= 4.5 CovGraphFact= 4.5 #CovGraphFact= 3.0 +<<<<<<< Updated upstream PartitionCount= 64 #PartitionCount= 8 #PartitionCount= 512 @@ -132,6 +138,13 @@ GSP2{ #PartitionCount= 32 #PartitionCount= 64 #PartitionCount= 32 +======= + #PartitionCount= 64 + PartitionCount= 256 + PartitionCountX= 8 + PartitionCountY= 8 + PartitionCountZ= 4 +>>>>>>> Stashed changes #PartitionCount= 3200 #PartitionCount= 6400 #PartitionCount= 1 @@ -200,7 +213,7 @@ GPMD{ #SMDAtomIndex1= 324 #SMDAtomIndex2= 315 UseSMD= F - HtoD= T + HtoD= F UseVectSKBlock= F ReplicateX= 1 ReplicateY= 1 diff --git a/examples/gpmdk/run/water/gpmd_input.in b/examples/gpmdk/run/water/gpmd_input.in index 2ff3af1d..1ab0f32a 100644 --- a/examples/gpmdk/run/water/gpmd_input.in +++ b/examples/gpmdk/run/water/gpmd_input.in @@ -14,12 +14,11 @@ Latte{ MDim= -1 #Threshold= 1.0d-5 Threshold= 0.0 - Verbose= 10 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) + Verbose= 5 #Verbosity levels: Basic info(0), 1(Basic routines info), 2(Print Physics data), 3(Print Relevant Matrices), 5(Print auxiliary matrices), 10(Print all) #SCF variables# - StopAt= "gpmdcov_Energ" + #StopAt= "gpmdcov_Energ" #StopAt= "gpmdcov_DM_Min" #StopAt= "gpmdcov_FirstCharges" - CoordsFile= "CH4.pdb" MPulay= 10 #ZMat= ZSP ZMat= Diag @@ -30,13 +29,57 @@ Latte{ MaxSCFIter= 500 CoulAcc= 1.0d-5 TimeRatio= 10.0 - TimeStep= 0.1 + TimeStep= 0.2 #TimeStep= 0.00 - MDSteps= 200000 - ParamPath= "../sulfurTBparam" + MDSteps= 10000 + #ParamPath= "../sulfurTBparam" + ParamPath= "../../tests/latteTBparams" + #ParamPath= "../latteTBparams_orig" + #CoordsFile= coords.ltt + #CoordsFile= coords_300New.dat + #CoordsFile= coords_300_sort.dat + CoordsFile= coords_300.dat + #CoordsFile= coords_2088.dat + #CoordsFile= "./polyaniline.pdb" + #CoordsFile= graphite2048.pdb + #CoordsFile= carbon_2197.pdb + #CoordsFile= ammonia2889.pdb + #CoordsFile= amonia1583.pdb + #CoordsFile= ammonia9863.pdb + #CoordsFile= ammonia4071.pdb + #CoordsFile= ammonia6610.pdb + #CoordsFile= Tr_Cage_opt.pdb + #CoordsFile= Tr_Cage.pdb + #CoordsFile= water3.pdb + #CoordsFile= carbon.pdb + #CoordsFile= carbon343.pdb + #CoordsFile= O2_solv114.pdb + #CoordsFile= full_tol360.pdb + #CoordsFile= full.pdb + #CoordsFile= NCH4.dat + #CoordsFile= full_tol3240.pdb + #CoordsFile= full_tol1440.pdb + #CoordsFile= water_box.pdb + #CoordsFile= coords_100WatAnders.dat + #CoordsFile= rand_H2O_1000.dat + #CoordsFile= coords_1032.dat + #CoordsFile= coords_2088.dat + #CoordsFile= coords_2955.pdb + #CoordsFile= amonia333.pdb + #CoordsFile= ammonia85.pdb + #CoordsFile= water3.dat + #CoordsFile= coords_10000.pdb + #CoordsFile= coords_6495.pdb + #CoordsFile= coords_30000.pdb + #CoordsFile= coords_65000.pdb + #CoordsFile= coords_100000.pdb NlistEach= 1 + #BMLDistributionType= Distributed MuCalcType= FromParts + #MuCalcType= Combined + #MuCalcType= Dyn EFermi= -0.0 + #kBT= 0.04308695 kBT= 0.025 Entropy= T DoKernel= F @@ -56,8 +99,6 @@ GSP2{ BMLType= Ellpack #GraphElement= Orbital GraphElement= Atom - #GraphElement= Orbital - #PartitionType= Block #NodesPerPart= 333 #NodesPerPart= 18 #NodesPerPart= 27 @@ -66,33 +107,29 @@ GSP2{ #NodesPerPart= 48 #NodesPerPart= 150 #NodesPerPart= 1331 - #PartitionType= METIS - PartitionType= Block - NodesPerPart= 0 + PartitionType= Sedacs #PartitionType= METIS+SA #PartitionType= METIS+KL #PartitionRefinement= None #PartitionRefinement= KL #PartitionRefinement= SA - NLGraphCut= 4.5 - CovGraphFact= 4.5 + NLGraphCut= 3.0 + CovGraphFact= 3.0 #CovGraphFact= 3.0 #PartitionCount= 128 - #PartitionCount= 4 - PartitionCount= 1 + #PartitionCount= 256 #PartitionCount= 512 #PartitionCount= 16 - #PartitionCount= 2 + PartitionCount= 16 #PartitionCount= 8 #PartitionCount= 1024 #PartitionCount= 32 - #PartitionCount= 64 #PartitionCount= 3200 #PartitionCount= 6400 #PartitionCount= 1 - GraphThreshold= 0.005 + GraphThreshold= 0.05 ErrLimit= 1.0e-12 - PartEach= 100 + PartEach= 3 Mdim= -1 } @@ -121,8 +158,10 @@ XLBO{ KERNEL{ KernelType= ByParts + #KernelType= Full + #KernelType= ByBlocks BuildAlways= F - RankNUpdate= 3 + RankNUpdate= 4 KernelMixing= F InitialMixingWith= DIIS UpdateEach= 1 @@ -134,20 +173,9 @@ GPMD{ DoVelocityRescale= F #VRFactor= 1.0 WriteTrajectory= T - WriteCoordsEach= 1 - #TrajectoryFormat= XYZ - InitialTemperature= 300 - #HtoD= T - MinimizationSteps= 100 - LangevinDynamics= T + WriteCoordsEach= 10 LangevinMethod= Siva - UseSMD= F - #For polyaniline - #SMDAtomIndex1= 6 - #SMDAtomIndex2= 7 - - SMDForceConstant= -8.0 - SMDR0= 0.001 - ComputeCurrents= T - CurrentThreshold= 0.0 + LangevinDynamics= F + LangevinGamma= 0.01 + InitialTemperature= 300.0 } diff --git a/examples/gpmdk/run/water/input.in b/examples/gpmdk/run/water/input.in index 7d11e51c..92be1dd9 100644 --- a/examples/gpmdk/run/water/input.in +++ b/examples/gpmdk/run/water/input.in @@ -108,7 +108,11 @@ GSP2{ #NodesPerPart= 48 #NodesPerPart= 150 #NodesPerPart= 1331 +<<<<<<< Updated upstream PartitionType= METIS +======= + PartitionType= Sedacs +>>>>>>> Stashed changes #PartitionType= METIS+SA #PartitionType= METIS+KL #PartitionRefinement= None @@ -121,8 +125,13 @@ GSP2{ #PartitionCount= 256 #PartitionCount= 512 #PartitionCount= 16 +<<<<<<< Updated upstream #PartitionCount= 2 PartitionCount= 8 +======= + PartitionCount= 8 + #PartitionCount= 8 +>>>>>>> Stashed changes #PartitionCount= 1024 #PartitionCount= 32 #PartitionCount= 3200 diff --git a/examples/gpmdk/run_lib/main_field_induced_forces.py b/examples/gpmdk/run_lib/main_field_induced_forces.py index ee8d48b3..de727e92 100755 --- a/examples/gpmdk/run_lib/main_field_induced_forces.py +++ b/examples/gpmdk/run_lib/main_field_induced_forces.py @@ -8,29 +8,204 @@ import scipy.linalg as sp from gpmd import * +def get_dipole(coords,charges): + dipole = np.zeros(3) -for i in range(1): - #GPMD verbosity level. - verb = 0 - #Load the coordinates and atom types - #latticeVectors,symbols,atomTypes,coords = read_xyz_file("H2.xyz",lib="None",verb=True) - latticeVectors,symbols,atomTypes,coords = read_pdb_file("benzene-CN.pdb",lib="None",verb=True) - #latticeVectors,symbols,atomTypes,coords = read_pdb_file("H2O.pdb",lib="None",verb=True) - print("Processing molecule H2O") + for i in range(len(charges)): + dipole[0] = dipole[0] + charges[i]*coords[i,0] + dipole[1] = dipole[1] + charges[i]*coords[i,1] + dipole[2] = dipole[2] + charges[i]*coords[i,2] + + +def get_field_born_charges(latticeVectors,symbols,atomTypes,coords,verb): + + field = np.zeros(3) + err,charges_out,forces_saved,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + nats = len(atomTypes) + bornField = np.zeros((nats,3,3)) + for i in range(3): + + field = np.zeros(3) + field[i] = -1.0E-4 + err,charges_out,forces_out,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + bornField[:,:,i] = (forces_out - forces_saved)/np.linalg.norm(field) + + field = np.zeros(3) + field[i] = 1.0E-4 + err,charges_out,forces_out,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + bornField[:,:,i] = (forces_out - forces_saved)/np.linalg.norm(field) + + + bchMat = np.zeros((3,3)) + bornCharges = np.zeros((nats)) + for i in range(len(bornField[:,0,0])): + bchMat[:,:] = bornField[i,:,:] + E,Q = sp.eigh(bchMat) + bornCharges[i] = sum(E)/3.0 + + return bornCharges + + +def get_displ_born_charges(latticeVectors,symbols,atomTypes,coordsIn,verb): - #Call the general gpmd funtion + nats = len(atomTypes) + bornDispl = np.zeros((nats,3,3)) + dspl = 1.0E-7 field = np.zeros((3)) - field[0] = 0.0 ; field[1] = 0.0 ; field[2] = 0.0 - err,charges_out,forces_out,dipole_out = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords = np.copy(coordsIn) + err,charges_out,forces_out,dipole_saved, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + for j in range(nats): + coords[j,0] = coordsIn[j,0] + dspl + err,charges_out,forces_out,dipole_out_p, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,0] = coordsIn[j,0] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + bornDispl[j,:,0] = (dipole_out_p - dipole_out_m)/(2*dspl) + coords[j,0] = coordsIn[j,0] + dspl + + for j in range(nats): + coords[j,1] = coordsIn[j,1] + dspl + err,charges_out,forces_out,dipole_out_p,energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,1] = coordsIn[j,1] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,1] = coordsIn[j,1] + dspl + bornDispl[j,:,1] = (dipole_out_p - dipole_out_m)/(2*dspl) + for j in range(nats): + coords[j,2] = coordsIn[j,2] + dspl + err,charges_out,forces_out,dipole_out_p, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,2] = coordsIn[j,2] - 2*dspl + err,charges_out,forces_out,dipole_out_m, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + coords[j,2] = coordsIn[j,2] + dspl + bornDispl[j,:,2] = (dipole_out_p - dipole_out_m)/(2*dspl) + + + bchMat = np.zeros((3,3)) + bornCharges = np.zeros((nats)) + for i in range(len(bornDispl[:,0,0])): + bchMat[:,:] = bornDispl[i,:,:] + E,Q = sp.eigh(bchMat) + bornCharges[i] = sum(E)/3.0 + + return bornCharges, bornDispl + + + +############################### +## Main starts here +############################### + +#****************************** +#Check forces with no Field +#****************************** +verb = 0 +#Load the coordinates and atom types +latticeVectors,symbols,atomTypes,coords0 = read_pdb_file("H2O.pdb",lib="None",verb=True) + +nats = len(atomTypes) +dspl = 0.00001 +field = np.zeros((3)) +field[0] = 1.0 + +#Get forces at coords0 +err,charges_out_pl,forces0,dipole_out, energyp = gpmd(latticeVectors,symbols,atomTypes,coords0,field,verb) - print("Charge on atom 1",charges_out[0]) - print("Dipole",dipole_out) - print("Forces on atom 1",forces_out[0,:]) +coordsP = np.copy(coords0) +coordsP[0,0] = coords0[0,0] + dspl +err,charges_out_pl,forces_out_pl,dipole_out, energyp = gpmd(latticeVectors,symbols,atomTypes,coordsP,field,verb) + +coordsM = np.copy(coords0) +coordsM[0,0] = coords0[0,0] - dspl +err,charges_out_pl,forces_out_pl,dipole_out, energym = gpmd(latticeVectors,symbols,atomTypes,coordsM,field,verb) + +print("Check: dE/dx,F",(energyp - energym)/(2*dspl),forces0[0,0]) + +#****************************** +#Compute forces with field +#****************************** + +field = np.zeros((3)) +#lambdaF = 1000000.0 +lambdaF = 0.000001 +field[0] = -lambdaF +coordsM = np.copy(coords0) +err,charges,forcesm,dipole, energy = gpmd(latticeVectors,symbols,atomTypes,coordsM,field,verb) + +field[0] = lambdaF +coordsP = np.copy(coords0) +err,charges,forcesp,dipole, energy = gpmd(latticeVectors,symbols,atomTypes,coordsP,field,verb) + + +bornCh = (forcesp - forcesm)/(2*lambdaF) + + +#bornChDiag,bornChargesDspl = get_displ_born_charges(latticeVectors,symbols,atomTypes,coords0,verb) + +for j in range(len(atomTypes)): + #print("Ch,BChD,BChF",j,bornChargesDspl[j,0,0],bornCh[j,0]) + print("Ch,BChD,BChF",j,bornCh[j,0]) + print("Fp,Fm",j,forcesp[0,0]-forcesm[0,0]) + + +exit(0) +#latticeVectors,symbols,atomTypes,coords = read_pdb_file("H2O.pdb",lib="None",verb=True) + +#Call the general gpmd funtion +field = np.zeros((3)) +dipole_sved = np.zeros((3)) +charges_saved = np.zeros((nats)) +field[0] = 0.0 ; field[1] = 0.001 ; field[2] = 0.0 +err,charges_out,forces_out,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) +charges_saved[:] = charges_out[:] + +forces_saved = np.zeros((nats,3)) +forces_saved[:] = forces_out[:] +print("Charge on atom 1",charges_out[0]) +print("Dipole",dipole_out) +#dipole_out = get_dipole(coords,charges_out) +#print("Dipole",dipole_out) +print("Forces on atom 1",forces_out[0,:]) +dipole_saved = np.zeros((3)) +dipole_saved[:] = dipole_out[:] + +field[0] = 0.0 ; field[1] = -0.001 ; field[2] = 0.0 +err,charges_out,forces_out,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) +print("charges_out",charges_out) + +nats = len(coords[:,0]) +bornField = np.zeros((nats,3)) +bornField = forces_out - forces_saved +bornField = bornField/np.linalg.norm(field) +print("bornField",bornField[9,1],charges_out[9],charges_saved[9]) +dx = 0.001 +field[:] = 0.0 +bornDispl = np.zeros((3)) +deltaQ = np.zeros((nats)) +sumBF = 0.0 + +for j in range(nats): + coords[j,0] = coords[j,0] + dx + err,charges_out,forces_out,dipole_out, energy = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) + mydip = np.zeros((3)) + for k in range(nats): + mydip[:] = mydip[:] + (charges_out[k]-charges_saved[k])*coords[k,:] + + + print("mydip",mydip) + print("dipole",dipole_out) + print("dipole_saved",dipole_saved) + print("dipole_diff",dipole_out - dipole_saved) + print("charges_diff",charges_out - charges_saved) + bornDispl = (dipole_out - dipole_saved)/dx + print("BornDispl",bornDispl[0]) + print("BornField",bornField[j,0]) + sumBF = sumBF + bornField[j,0] + deltaQ = charges_out - charges_saved + print("deltaQ",charges_out[j],deltaQ[j],deltaQ[j]*(coords[j,0] - dx) + charges_out[j]) + print("BornBoth",charges_out[j],bornDispl[0],bornField[j,0],bornDispl[0]/bornField[j,0]) + exit(0) + coords[j,0] = coords[j,0] - dx - field[0] = 0.01 ; field[1] = 0.0 ; field[2] = 0.0 - err,charges_out,forces_out,dipole_out = gpmd(latticeVectors,symbols,atomTypes,coords,field,verb) - print("Forces on atom 1",forces_out[0,:]) +print("SumBF",sumBF) # tch = 0.0 # bchMat = np.zeros((3,3)) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index d34fa108..b10f7e20 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -142,6 +142,10 @@ subroutine gpmdcov_EnergAndForces(charges) call prg_get_pulayforce(syprt(ipt)%nats,syprt(ipt)%estr%zmat,syprt(ipt)%estr%ham,syprt(ipt)%estr%rho,& dSx_bml,dSy_bml,dSz_bml,syprt(ipt)%estr%hindex,syprt(ipt)%estr%FPUL,lt%threshold) + !call prg_PulayComponentT(syprt(ipt)%estr%rho,syprt(ipt)%estr%ham,syprt(ipt)%estr%zmat,syprt(ipt)%estr%FPUL,lt%threshold & + ! &,lt%mdim,lt%bml_type,lt%verbose) + + call get_nonortho_coul_forces(syprt(ipt)%nats, norb, dSx_bml,dSy_bml,dSz_bml,& syprt(ipt)%estr%hindex,syprt(ipt)%spindex,syprt(ipt)%estr%rho,syprt(ipt)%net_charge,syprt(ipt)%estr%coul_pot_r,& syprt(ipt)%estr%coul_pot_k,tb%hubbardu,syprt(ipt)%estr%FSCOUL,lt%threshold) @@ -171,6 +175,7 @@ subroutine gpmdcov_EnergAndForces(charges) enddo collectedforce = GFPUL + GFSCOUL + SKForce + !collectedforce = GFSCOUL + SKForce call gpmdcov_msMem("gpmdcov","Before steered MD (SMD) check",lt%verbose,myRank) @@ -309,7 +314,7 @@ subroutine gpmdcov_EnergAndForces(charges) EPOT = Etot + entropy - if(myRank == 1 .and. lt%verbose >= 2)then + if((myRank == 1) .and. (lt%verbose >= 2))then write(*,*)"Energy Coulomb = ", ECoul write(*,*)"Energy Band =", sum(ebandvector(:)) write(*,*)"Energy Repulsive = ", ERep @@ -319,7 +324,18 @@ subroutine gpmdcov_EnergAndForces(charges) if(.not.allocated(sy%force))allocate(sy%force(3,sy%nats)) + !TOTAL FORCES sy%force = collectedforce + PairForces + coul_forces + !sy%force = SKForce + GFSCOUL + GFPUL + PairForces + coul_forces + !sy%force = SKForce + GFSCOUL + GFPUL PairForces + coul_forces + !sy%force = coul_forces + write(*,*)"FORCESSS",sy%force + !sy%force = SKForce + GFSCOUL + PairForces + coul_forces + !sy%force = GFSCOUL + !sy%force = SKForce + GFSCOUL + coul_forces + PairForces + !write(*,*)"FORCES!!!",SKForce,GFSCOUL,coul_forces + !sy%force = collectedforce + !sy%force = coul_forces if(myRank == 1 .and. lt%verbose >= 3)then write(*,*)""; write(*,*)"FPUL + FSCOUL + SKForce" diff --git a/examples/gpmdk/src/gpmdcov_init.F90 b/examples/gpmdk/src/gpmdcov_init.F90 index b52986cb..1f6265a6 100644 --- a/examples/gpmdk/src/gpmdcov_init.F90 +++ b/examples/gpmdk/src/gpmdcov_init.F90 @@ -164,9 +164,9 @@ subroutine gpmdcov_Init(lib_on) !> Center sytem inside the box and fold it by the lattice_vectors. This is ! done only for visualization purposes. !origin = 0.0_dp - if(gpmdt%trfl)then - call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) - endif + !if(gpmdt%trfl)then + ! call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) + !endif !origin = 0.0_dp if(gpmdt%restartfromdump)then diff --git a/src/prg_graph_mod.F90 b/src/prg_graph_mod.F90 index fe31445e..35be9ac7 100644 --- a/src/prg_graph_mod.F90 +++ b/src/prg_graph_mod.F90 @@ -637,7 +637,7 @@ subroutine do_flips_precomp(whichParts,coords,graph,adj,degs,nnodes,nparts,bal) !Actualizing possible cuts of neighbors of i and j do ii = 1,degs(i) ind = graph(i,ii) - !The nighs of i "if now they are in the "new color + !The neighs of i "if now they are in the "new color !of i", their cut "at that color" will be decreased !by one. cutsI(ind,partIndexJ) = cutsI(ind,partIndexJ) - 1 diff --git a/src/prg_partition_mod.F90 b/src/prg_partition_mod.F90 index 873af43e..9dd3ae98 100644 --- a/src/prg_partition_mod.F90 +++ b/src/prg_partition_mod.F90 @@ -40,6 +40,7 @@ module prg_partition_mod public :: prg_update_gp public :: prg_simAnnealing_old + #ifdef DO_GRAPHLIB interface From 769d607fe505a5655f29921b535f058ffaa4f054 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 12 Aug 2024 17:01:04 -0600 Subject: [PATCH 36/69] Attempt to fix the hackathon branch --- examples/gpmdk/src/gpmdcov_init.F90 | 6 +- examples/gpmdk/src/gpmdcov_part.F90 | 85 +++++++++++++++++++++-------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_init.F90 b/examples/gpmdk/src/gpmdcov_init.F90 index 1f6265a6..b52986cb 100644 --- a/examples/gpmdk/src/gpmdcov_init.F90 +++ b/examples/gpmdk/src/gpmdcov_init.F90 @@ -164,9 +164,9 @@ subroutine gpmdcov_Init(lib_on) !> Center sytem inside the box and fold it by the lattice_vectors. This is ! done only for visualization purposes. !origin = 0.0_dp - !if(gpmdt%trfl)then - ! call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) - !endif + if(gpmdt%trfl)then + call prg_translateandfoldtobox(sy%coordinate,sy%lattice_vector,origin) + endif !origin = 0.0_dp if(gpmdt%restartfromdump)then diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index 8b9eeafd..4eddd4fa 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -14,7 +14,7 @@ subroutine gpmdcov_Part(ipreMD) integer, allocatable :: graph_h(:,:) integer, allocatable :: graph_p(:,:) integer, allocatable, save :: graph_p_old(:,:) - integer, allocatable, save :: G_added(:,:), G_removed(:,:), G_updated(:,:) + integer, allocatable, save :: G_added(:,:), G_removed(:,:), G_updated(:,:), G_updated_mp(:,:) integer, allocatable, save :: N_added(:), N_removed(:), NNZ1(:), NNZ2(:), NNZ_updated(:) logical, allocatable, save :: v(:), v_check(:), check_graph integer :: n_atoms, max_updates, k, ktot @@ -46,16 +46,17 @@ subroutine gpmdcov_Part(ipreMD) if(.not.allocated(graph_p_old))then allocate(graph_p_old(myMdim,n_atoms)) graph_p_old = 0 - allocate(v(n_atoms)) - allocate(v_check(n_atoms)) allocate(G_added(max_updates,n_atoms)) allocate(G_removed(max_updates,n_atoms)) allocate(G_updated(max_updates,n_atoms)) + allocate(G_updated_mp(max_updates,n_atoms)) allocate(N_added(n_atoms)) allocate(N_removed(n_atoms)) allocate(NNZ1(n_atoms)) allocate(NNZ2(n_atoms)) allocate(NNZ_updated(n_atoms)) + allocate(v(n_atoms)) + allocate(v_check(n_atoms)) endif #endif !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -259,19 +260,29 @@ subroutine gpmdcov_Part(ipreMD) ! % Check NNZ_Updated: G_Updated = G2 But edges are not in the same order write(*,*)"DEBUG: ktot = ",ktot - call prg_sumIntReduceN(graph_p, myMdim*sy%nats) + !call prg_sumIntReduceN(graph_p, myMdim*sy%nats) write(*,*)"DEBUG: Doing graph update reduction at mdstep ",mdstep ! %% Use G_removed and G_added to update from G1 to G2 call prg_sumIntReduceN(G_added,n_atoms*max_updates) call prg_sumIntReduceN(G_removed,n_atoms*max_updates) G_updated = 0 + G_updated_mp = 0 NNZ_updated = 0 - v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. + v = .false. + v_check = .false. ! do iipt=1,partsInEachRank(myRank) ! ipt= reshuffle(iipt,myRank) ! do ii = 1,gpat%sgraph(ipt)%llsize ! i = gpat%sgraph(ipt)%core_halo_index(ii) + 1 + !$omp parallel do & + !$omp default(none) & + !$omp private(i,j,k) & + !$omp firstprivate(v,v_check) & + !$omp shared(G_added,G_removed,G_updated_mp,N_added,N_removed) & + !$omp shared(graph_p_old,graph_p,NNZ1,NNZ2,NNZ_updated,n_atoms) & + !$omp shared(check_graph) do i = 1,n_atoms + v = .false. do j = 1,NNZ1(i) v(graph_p_old(j,i)) = .true. end do @@ -282,31 +293,57 @@ subroutine gpmdcov_Part(ipreMD) do j = 1,NNZ1(i) if (v(graph_p_old(j,i)) .eqv. .true.)then ! % Account only for the remaining edges k = k + 1; - G_updated(k,i) = graph_p_old(j,i); + G_updated_mp(k,i) = graph_p_old(j,i); end if NNZ_updated(i) = k + N_added(i); end do do j = k+1,NNZ_updated(i) - G_updated(j,i) = G_added(j-k,i) ! Add new edges at the end + G_updated_mp(j,i) = G_added(j-k,i) ! Add new edges at the end end do - check_graph = .true. - if (check_graph)then - v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. - v_check = .false. - if (NNZ_updated(i) == NNZ2(i))then - write(*,*)"DEBUG: Number of nonzero elements is the same" - end if - do j = 1,NNZ2(i) - v(graph_p(j,i)) = .true. - v_check(G_updated(j,i)) = .true. - end do - if (all(v.eqv.v_check))then - write(*,*)"DEBUG: elements are the same" - end if - end if - end do + end do + !$omp end parallel do + ! do i = 1,n_atoms + ! v = .false. + ! do j = 1,NNZ1(i) + ! v(graph_p_old(j,i)) = .true. + ! end do + ! do j = 1,N_removed(i) + ! v(G_removed(j,i)) = .false. ! % Remove edges + ! end do + ! k = 0 + ! do j = 1,NNZ1(i) + ! if (v(graph_p_old(j,i)) .eqv. .true.)then ! % Account only for the remaining edges + ! k = k + 1; + ! G_updated(k,i) = graph_p_old(j,i); + ! end if + ! NNZ_updated(i) = k + N_added(i); + ! end do + ! do j = k+1,NNZ_updated(i) + ! G_updated(j,i) = G_added(j-k,i) ! Add new edges at the end + ! end do + ! check_graph = .false. + ! if (check_graph)then + ! v = .false. ! % Temporary vector that keeps track of elements that are there and then removed. + ! v_check = .false. + ! if (NNZ_updated(i) == NNZ2(i))then + ! write(*,*)"DEBUG: Number of nonzero elements is the same" + ! end if + + ! do j = 1,NNZ2(i) + ! v(graph_p(j,i)) = .true. + ! v_check(G_updated(j,i)) = .true. + ! end do + ! if (all(v.eqv.v_check))then + ! write(*,*)"DEBUG: elements are the same" + ! end if + ! end if + ! end do + ! if(.not.all(G_updated.eq.G_updated_mp))then + ! write(*,*)"DEBUG: Updated graph differs when using omp parallel do" + ! endif + !!$omp end parallel do !end do - graph_p = G_updated + graph_p = G_updated_mp graph_p_old = graph_p endif ! call prg_sumIntReduceN(auxVectInt, myMdim*sy%nats) From 6a2cb5f981dd9c43d0c3c4083cae51f0ab8a7ba3 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 12 Aug 2024 17:04:55 -0600 Subject: [PATCH 37/69] Eliminate debug output of FORCESS --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index b10f7e20..9f005607 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -329,7 +329,7 @@ subroutine gpmdcov_EnergAndForces(charges) !sy%force = SKForce + GFSCOUL + GFPUL + PairForces + coul_forces !sy%force = SKForce + GFSCOUL + GFPUL PairForces + coul_forces !sy%force = coul_forces - write(*,*)"FORCESSS",sy%force + !write(*,*)"FORCESSS",sy%force !sy%force = SKForce + GFSCOUL + PairForces + coul_forces !sy%force = GFSCOUL !sy%force = SKForce + GFSCOUL + coul_forces + PairForces From 6722bda61aae372a643276f5c615131f5a111dba Mon Sep 17 00:00:00 2001 From: "Michael E. Wall" Date: Tue, 13 Aug 2024 10:02:16 -0600 Subject: [PATCH 38/69] Update bml submodule --- bml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bml b/bml index d2d85933..c5b1986e 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit d2d8593313caccf9ea7933e76d2dc9db718bd6bd +Subproject commit c5b1986e156c23e687ce018470c06f9813bb25de From 87cf0c8775ee5e3c2277177bf73bd4fa243633e3 Mon Sep 17 00:00:00 2001 From: "Michael E. Wall" Date: Tue, 13 Aug 2024 10:10:34 -0600 Subject: [PATCH 39/69] Update bml submodule and move build scripts to scripts/ dir --- bml | 2 +- .../build_chicoma_hackathon.sh | 0 build_progress_nvhpc.sh => scripts/build_progress_nvhpc.sh | 0 build_venado_hackathon.sh => scripts/build_venado_hackathon.sh | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename build_chicoma_hackathon.sh => scripts/build_chicoma_hackathon.sh (100%) rename build_progress_nvhpc.sh => scripts/build_progress_nvhpc.sh (100%) rename build_venado_hackathon.sh => scripts/build_venado_hackathon.sh (100%) diff --git a/bml b/bml index c5b1986e..1dd97873 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit c5b1986e156c23e687ce018470c06f9813bb25de +Subproject commit 1dd97873271f402bc85fe41f51de3fd9f34602d2 diff --git a/build_chicoma_hackathon.sh b/scripts/build_chicoma_hackathon.sh similarity index 100% rename from build_chicoma_hackathon.sh rename to scripts/build_chicoma_hackathon.sh diff --git a/build_progress_nvhpc.sh b/scripts/build_progress_nvhpc.sh similarity index 100% rename from build_progress_nvhpc.sh rename to scripts/build_progress_nvhpc.sh diff --git a/build_venado_hackathon.sh b/scripts/build_venado_hackathon.sh similarity index 100% rename from build_venado_hackathon.sh rename to scripts/build_venado_hackathon.sh From 63828d5597e062fa8d674a05f67ad7735262b55f Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 22 Aug 2024 16:16:39 -0600 Subject: [PATCH 40/69] breaking lines --- examples/gpmdk/src/gpmdcov_nvtx.F90 | 4 +++- examples/gpmdk/src/gpmdcov_response.F90 | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_nvtx.F90 b/examples/gpmdk/src/gpmdcov_nvtx.F90 index cfa2f716..e0520a31 100644 --- a/examples/gpmdk/src/gpmdcov_nvtx.F90 +++ b/examples/gpmdk/src/gpmdcov_nvtx.F90 @@ -2,7 +2,9 @@ module gpmdcov_nvtx_mod use iso_c_binding implicit none - integer(4),private :: col(7) = [ int(Z'0000ff00',4), int(Z'000000ff',4), int(Z'00ffff00',4), int(Z'00ff00ff',4), int(Z'0000ffff',4), int(Z'00ff0000',4), int(Z'00ffffff',4)] + integer(4),private :: col(7) = [ int(Z'0000ff00',4), int(Z'000000ff',4), & + & int(Z'00ffff00',4), int(Z'00ff00ff',4), int(Z'0000ffff',4), & + & int(Z'00ff0000',4), int(Z'00ffffff',4)] character(kind=c_char,len=256),private,target :: tempName type, bind(C):: nvtxEventAttributes diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 8d3016e9..1e5f0408 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -125,7 +125,8 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !$omp simd do j = 1,HDIM !P1(:,k) = 1.D0/(2.D0*p_0(:)*(p_0(:)-1.D0)+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) - P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & + &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) enddo !$omp end simd !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) From 52e7fd6eaa470c4eb4adcab5d8ee9e267420c36c Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 22 Aug 2024 17:07:32 -0600 Subject: [PATCH 41/69] fixing gpmd.py --- examples/gpmdk/src/gpmd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/gpmdk/src/gpmd.py b/examples/gpmdk/src/gpmd.py index 629cc35c..0105ba37 100755 --- a/examples/gpmdk/src/gpmd.py +++ b/examples/gpmdk/src/gpmd.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import numpy as np -from sdc_system import * -from sdc_ptable import ptable +from sedacs.system import * +from sedacs.periodic_table import PeriodicTable import ctypes as ct import os @@ -20,13 +20,13 @@ def gpmd(latticeVectors,symbols,atomTypes,coords,field,verb): # Import the shared library - gpmdLibFileName = os.environ['GPMD_PATH'] + '/libgpmd.so' + gpmdLibFileName = os.environ['GPMD_PATH'] + '/libgpmdk.so' gpmdLib = ct.CDLL(gpmdLibFileName) f = gpmdLib.gpmd_compute #Periodic table: We use this to pass the chemical atom types as integer instead of characters. - pt = ptable() + pt = PeriodicTable() nats = len(coords[:,1]) From 23c3af7ec6cbd74fea1aa6c320329531abd3d238 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 23 Aug 2024 15:03:51 -0600 Subject: [PATCH 42/69] Add more nvtx tags --- bml | 2 +- examples/gpmdk/src/gpmdcov_diagonalize.F90 | 25 ++++++++++++++ examples/gpmdk/src/gpmdcov_dm_min.F90 | 16 ++++++++- examples/gpmdk/src/gpmdcov_energandforces.F90 | 34 +++++++++++++++---- examples/gpmdk/src/gpmdcov_initparts.F90 | 7 ++++ 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/bml b/bml index 1dd97873..a4dd0321 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit 1dd97873271f402bc85fe41f51de3fd9f34602d2 +Subproject commit a4dd0321ef7df6f227c540327d2c4647cc7ceac0 diff --git a/examples/gpmdk/src/gpmdcov_diagonalize.F90 b/examples/gpmdk/src/gpmdcov_diagonalize.F90 index 09ec1429..6fc9363f 100644 --- a/examples/gpmdk/src/gpmdcov_diagonalize.F90 +++ b/examples/gpmdk/src/gpmdcov_diagonalize.F90 @@ -5,6 +5,10 @@ module gpmdcov_diagonalize_mod use gpmdcov_mod use gpmdcov_writeout_mod use prg_extras_mod + +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif contains @@ -115,6 +119,13 @@ subroutine gpmdcov_Diagonalize_H1(nguess) call bml_copy(syprt(ipt)%estr%ham0,syprt(ipt)%estr%ham) !> Get the scf hamiltonian. The output is ham_bml. call gpmdcov_msIII("gpmdcov_DM_Min","In prg_get_hscf...",lt%verbose,myRank) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierBeforePrgGetHscf",2) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif call prg_get_hscf(syprt(ipt)%estr%ham0,syprt(ipt)%estr%over,syprt(ipt)%estr%ham,syprt(ipt)%spindex,& &syprt(ipt)%estr%hindex,tb%hubbardu,syprt(ipt)%net_charge,& &syprt(ipt)%estr%coul_pot_r,syprt(ipt)%estr%coul_pot_k,lt%mdim,lt%threshold) @@ -123,6 +134,13 @@ subroutine gpmdcov_Diagonalize_H1(nguess) call gpmdcov_msMem("gpmdcov_Diagonalize_H1", "Before orth ",lt%verbose,myRank) if(bml_allocated(syprt(ipt)%estr%oham)) call bml_deallocate(syprt(ipt)%estr%oham) call bml_zero_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%oham) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierBeforeOrthogonalizeH1",3) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif !> Orthogonalize ham. call prg_orthogonalize(syprt(ipt)%estr%ham,syprt(ipt)%estr%zmat,syprt(ipt)%estr%oham,& @@ -145,6 +163,13 @@ subroutine gpmdcov_Diagonalize_H1(nguess) & gpat%sgraph(ipt)%llsize, syprt(ipt)%estr%evals, syprt(ipt)%estr%dvals, & & syprt(ipt)%estr%evects, lt%verbose) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierAfterDiagonalizeH1",4) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif norbsInEachCHAtRank(iptt) = size(syprt(ipt)%estr%evals,dim=1) call bml_deallocate(syprt(ipt)%estr%oham) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index c43a0575..e7ea770a 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -302,6 +302,13 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) !> Real contribution to the Coul energy. The outputs are !coul_forces_r,coul_pot_r. call gpmdcov_msI("gpmdcov_DM_Min","In real Coul ...",lt%verbose,myRank) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierBeforeEwald",2) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif call gpmdcov_msMem("gpmdcov_dm_min_eig","Before get_ewald_list_real_dcalc_vect",lt%verbose,myRank) if(myRank == 1 .and. lt%verbose >= 1) mls_coul = mls() @@ -324,6 +331,13 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) call gpmdcov_msII("gpmdcov_DM_Min","Time recip coul "//to_string(mls() - mls_coul)//" ms",lt%verbose,myRank) call gpmdcov_msMem("gpmdcov_dm_min_eig","After get_ewald_recip",lt%verbose,myRank) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierAfterEwald",3) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif if(iscf == Nr_SCF) converged = .true. call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before gpmd_diagonalize_H1",lt%verbose,myRank) @@ -335,7 +349,7 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) #ifdef DO_MPI #ifdef USE_NVTX - call nvtxStartRange("BarrierAfterH1",2) + call nvtxStartRange("BarrierAfterH1",4) call prg_barrierParallel call nvtxEndRange #endif diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 9f005607..6f52c858 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -2,6 +2,10 @@ module gpmdcov_EnergAndForces_mod use gpmdcov_mod +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif + contains subroutine gpmdcov_EnergAndForces(charges) @@ -103,6 +107,10 @@ subroutine gpmdcov_EnergAndForces(charges) dx = 0.0001_dp; +#ifdef USE_NVTX + call nvtxStartRange("get_dH_and_dS",2) +#endif + if(gpmdt%usevectsk)then call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& @@ -125,7 +133,10 @@ subroutine gpmdcov_EnergAndForces(charges) &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dSx_bml,dSy_bml,dSz_bml) endif - +#ifdef USE_NVTX + call nvtxEndRange +#endif + if(printRank() == 1 .and. lt%verbose >= 10)then call bml_print_matrix("dH0x_bml",dH0x_bml,0,10,0,10) call bml_print_matrix("dH0y_bml",dH0y_bml,0,10,0,10) @@ -134,14 +145,23 @@ subroutine gpmdcov_EnergAndForces(charges) call bml_print_matrix("dSx_bml",dSx_bml,0,10,0,10) call bml_print_matrix("dSy_bml",dSy_bml,0,10,0,10) call bml_print_matrix("dSz_bml",dSz_bml,0,10,0,10) - endif - + endif +#ifdef USE_NVTX + call nvtxStartRange("get_skforce",3) +#endif call get_skforce(syprt(ipt)%nats,syprt(ipt)%estr%rho,dH0x_bml,dH0y_bml,& dH0z_bml,syprt(ipt)%estr%hindex,syprt(ipt)%estr%SKForce,lt%threshold) - +#ifdef USE_NVTX + call nvtxEndRange + call nvtxStartRange("prg_get_pulayforce",4) +#endif + call prg_get_pulayforce(syprt(ipt)%nats,syprt(ipt)%estr%zmat,syprt(ipt)%estr%ham,syprt(ipt)%estr%rho,& dSx_bml,dSy_bml,dSz_bml,syprt(ipt)%estr%hindex,syprt(ipt)%estr%FPUL,lt%threshold) - +#ifdef USE_NVTX + call nvtxEndRange + call nvtxStartRange("get_nonortho_coul_forces",5) +#endif !call prg_PulayComponentT(syprt(ipt)%estr%rho,syprt(ipt)%estr%ham,syprt(ipt)%estr%zmat,syprt(ipt)%estr%FPUL,lt%threshold & ! &,lt%mdim,lt%bml_type,lt%verbose) @@ -149,7 +169,9 @@ subroutine gpmdcov_EnergAndForces(charges) call get_nonortho_coul_forces(syprt(ipt)%nats, norb, dSx_bml,dSy_bml,dSz_bml,& syprt(ipt)%estr%hindex,syprt(ipt)%spindex,syprt(ipt)%estr%rho,syprt(ipt)%net_charge,syprt(ipt)%estr%coul_pot_r,& syprt(ipt)%estr%coul_pot_k,tb%hubbardu,syprt(ipt)%estr%FSCOUL,lt%threshold) - +#ifdef USE_NVTX + call nvtxEndRange +#endif do i=1,gpat%sgraph(ipt)%llsize GFPUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FPUL(:,i) GFSCOUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FSCOUL(:,i) diff --git a/examples/gpmdk/src/gpmdcov_initparts.F90 b/examples/gpmdk/src/gpmdcov_initparts.F90 index e56e57ab..e0bbfe95 100644 --- a/examples/gpmdk/src/gpmdcov_initparts.F90 +++ b/examples/gpmdk/src/gpmdcov_initparts.F90 @@ -122,6 +122,13 @@ subroutine gpmdcov_InitParts call gpmdcov_buildz(syprt(ipt)%estr%over,syprt(ipt)%estr%zmat) if(lt%verbose >= 3) call prg_timer_stop(dyn_timer,1) +#ifdef DO_MPI +#ifdef USE_NVTX + call nvtxStartRange("BarrierAfterGenZ",4) + call prg_barrierParallel + call nvtxEndRange +#endif +#endif if(myRank == 1 .and. lt%verbose >= 5)then write(*,*)"Z matrix for part:" call bml_print_matrix("Z",syprt(ipt)%estr%zmat,0,6,0,6) From 69d937aaaf9b2f275cfa3c5be53fab298232a85a Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Mon, 26 Aug 2024 15:50:47 -0600 Subject: [PATCH 43/69] Added err_var --- examples/gpmdk/src/gpmdcov_buildz.F90 | 37 +- examples/gpmdk/src/gpmdcov_highorder.F90 | 68 +-- examples/gpmdk/src/gpmdcov_lib.F90 | 202 ++++---- examples/gpmdk/src/gpmdcov_lib_mod.F90 | 395 ++++++++-------- examples/gpmdk/src/gpmdcov_mod.F90 | 558 ++++++++++++----------- examples/gpmdk/src/gpmdcov_vars.F90 | 2 + examples/gpmdk/src/gpmdk.F90 | 1 + src/prg_genz_mod.F90 | 26 +- 8 files changed, 664 insertions(+), 625 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_buildz.F90 b/examples/gpmdk/src/gpmdcov_buildz.F90 index c4747e73..2f0f9625 100644 --- a/examples/gpmdk/src/gpmdcov_buildz.F90 +++ b/examples/gpmdk/src/gpmdcov_buildz.F90 @@ -1,25 +1,30 @@ +subroutine gpmdcov_buildz(overin_bml,zmatout_bml) + use gpmdcov_vars - subroutine gpmdcov_buildz(overin_bml,zmatout_bml) - use gpmdcov_vars + type(bml_matrix_t), intent(inout) :: zmatout_bml + type(bml_matrix_t), intent(inout) :: overin_bml - type(bml_matrix_t), intent(inout) :: zmatout_bml - type(bml_matrix_t), intent(inout) :: overin_bml + igenz = igenz + 1 - igenz = igenz + 1 + if(lt%zmat == "ZSP")then !Congruence transformation. - if(lt%zmat == "ZSP")then !Congruence transformation. + call prg_buildzsparse(overin_bml,zmatout_bml,igenz,lt%mdim,& + lt%bml_type, zk1_bml,zk2_bml,zk3_bml& + ,zk4_bml,zk5_bml,zk6_bml,zsp%nfirst,zsp%nrefi,zsp%nreff,& + zsp%numthresi,zsp%numthresf,zsp%integration,zsp%verbose) - call prg_buildzsparse(overin_bml,zmatout_bml,igenz,lt%mdim,& - lt%bml_type, zk1_bml,zk2_bml,zk3_bml& - ,zk4_bml,zk5_bml,zk6_bml,zsp%nfirst,zsp%nrefi,zsp%nreff,& - zsp%numthresi,zsp%numthresf,zsp%integration,zsp%verbose) - - else - - !Build Z matrix using diagonalization (usual method). - call prg_buildzdiag(overin_bml,zmatout_bml,lt%threshold,lt%mdim,lt%bml_type) + else + !Build Z matrix using diagonalization (usual method). + call prg_buildzdiag(overin_bml,zmatout_bml,lt%threshold,lt%mdim,lt%bml_type,0,err_status) + if(err_status)then + if(.not. lib_mode)then + stop "ERROR: Possible non positive definite overlap" + return + endif endif - end subroutine gpmdcov_buildz + endif + +end subroutine gpmdcov_buildz diff --git a/examples/gpmdk/src/gpmdcov_highorder.F90 b/examples/gpmdk/src/gpmdcov_highorder.F90 index 54298c47..ab1cdd80 100644 --- a/examples/gpmdk/src/gpmdcov_highorder.F90 +++ b/examples/gpmdk/src/gpmdcov_highorder.F90 @@ -17,7 +17,7 @@ module gpmdcov_highorder_mod ! Progress modules use prg_ptable_mod use prg_response_mod - + ! public :: gpmdcov_get_born_charges_v1 !public :: gpmdcov_get_born_charges_v2 @@ -25,14 +25,14 @@ module gpmdcov_highorder_mod contains #ifdef FULL - !> Computes the Born charges - !! \brief Based on the derivative of the dipole moment with respect - !! to the coordinates. - !! \param dipoleMoment Dipole moment of the molecule. - !! \param bornCharges for each atom - subroutine gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) - - implicit none + !> Computes the Born charges + !! \brief Based on the derivative of the dipole moment with respect + !! to the coordinates. + !! \param dipoleMoment Dipole moment of the molecule. + !! \param bornCharges for each atom + subroutine gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) + + implicit none real(dp),allocatable :: dipoleMoment(:),savedCharges(:),bornCharges(:,:) real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) real(dp) :: factor,dr,dfx,dfy,dfz @@ -114,11 +114,11 @@ subroutine gpmdcov_get_born_charges_v1(dipoleMoment,bornCharges) end subroutine gpmdcov_get_born_charges_v1 - !> Born charges - !! \brief Will compute the Born charges based on the derivatives - !! of the forces with respect to the field. + !> Born charges + !! \brief Will compute the Born charges based on the derivatives + !! of the forces with respect to the field. subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) - implicit none + implicit none real(dp),allocatable :: dipoleMoment(:),savedCharges(:),bornCharges(:,:) real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) real(dp) :: lambda,intensity,field(3),threshold @@ -130,12 +130,12 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) real(dp) :: intensity0,dintensity lambda = 1.0_dp !2.0_dp/14.3996437701414_dp - threshold = 0.0_dp + threshold = 0.0_dp verbose = 3 - + if(.not. allocated(bornCharges)) allocate(bornCharges(9,sy%nats)) - - !This will assume only one part + + !This will assume only one part call gpmdcov_InitParts() allocate(forcesSaved(3,sy%nats)) @@ -148,15 +148,16 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(1) = -1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) !call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ! ,prt_bml,threshold,sy%spindex,norbi,verbose) - sy%net_charge = 0.0_dp + ! ,prt_bml,threshold,sy%spindex,norbi,verbose) + sy%net_charge = 0.0_dp call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) call gpmdcov_Diagonalize_H0() call gpmdcov_muFromParts() + if(err_status)return call gpmdcov_FirstCharges(eig) call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) call gpmdcov_EnergAndForces(sy%net_charge) @@ -164,8 +165,8 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(1) = 1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) - sy%net_charge = 0.0_dp + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + sy%net_charge = 0.0_dp call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) @@ -182,7 +183,7 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(2) = -1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) @@ -195,7 +196,7 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(2) = 1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) @@ -212,7 +213,7 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(3) = -1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) @@ -225,7 +226,7 @@ subroutine gpmdcov_get_born_charges_v2(polarizability,bornCharges,norbi) field = 0.0_dp ; field(3) = 1.0_dp call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,intensity,sy%coordinate,lambda& - ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,threshold,sy%spindex,norbi,verbose,syprt(1)%estr%over) call gpmdcov_InitParts() call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,1.0_dp,lt%threshold) !H = H + V call bml_print_matrix("H",syprt(1)%estr%ham0,0,3,0,3) @@ -264,28 +265,29 @@ subroutine gpmdcov_get_field_forces(norbi,znucs,field_in) threshold = 0.0_dp verbose = 3 - kappa = 1.0_dp + kappa = 1.0_dp write(*,*)field_in field = field_in/norm2(field_in) intensity = norm2(field_in) - lambda = kappa*intensity + lambda = kappa*intensity write(*,*)(sy%coordinate) call bml_zero_matrix("dense",bml_element_real,dp,norb,norb,prt_bml) call prg_pert_constant_field(field,1.0_dp,sy%coordinate,1.0_dp& - ,prt_bml,0.0_dp,sy%spindex,norbi,verbose,syprt(1)%estr%over) + ,prt_bml,0.0_dp,sy%spindex,norbi,verbose,syprt(1)%estr%over) call bml_print_matrix("Pert",prt_bml,0,6,0,6) call bml_add(syprt(1)%estr%ham0,prt_bml,1.0_dp,-lambda,threshold) !H = H + lambda*V - + call gpmdcov_Diagonalize_H0() call gpmdcov_muFromParts() + if(err_status)return call gpmdcov_FirstCharges(eig) call gpmdcov_DM_Min_Eig(1000,sy%net_charge,.true.,.false.) call gpmdcov_EnergAndForces(sy%net_charge) - - do k = 1,sy%nats - sy%force(:,k) = sy%force(:,k) + field_in(:)*sy%net_charge(k) + + do k = 1,sy%nats + sy%force(:,k) = sy%force(:,k) + field_in(:)*sy%net_charge(k) enddo end subroutine gpmdcov_get_field_forces diff --git a/examples/gpmdk/src/gpmdcov_lib.F90 b/examples/gpmdk/src/gpmdcov_lib.F90 index a25b0033..b3aba958 100644 --- a/examples/gpmdk/src/gpmdcov_lib.F90 +++ b/examples/gpmdk/src/gpmdcov_lib.F90 @@ -1,17 +1,17 @@ !> Library interface -!! \brief This file is used to interface to python via iso_c_binding -!! library. +!! \brief This file is used to interface to python via iso_c_binding +!! library. !! \param nats Number of total atoms in the system -!! \param nTypes Number of atom types +!! \param nTypes Number of atom types !! \param coords_in Coordinates of every atom in the system. !! Allocation: !! \verbatim coordinate(3,nats) \endverbatim -!! \param latticeVectors_in Flattened lattice vectors/box +!! \param latticeVectors_in Flattened lattice vectors/box !! Allocation: !! \verbatim lattice_vector(3*3) \endverbatim !! \verbatim v1 = lattice_vector(1:3) \endverbatim !! \param atomTypes_in Atom type index for every atom -!! It gives the species index of a particulat atom. Indexing starts from 0! +!! It gives the species index of a particulat atom. Indexing starts from 0! !! Allocation: !! \verbatim atomTypes(nats) \endverbatim !! If we need the index of atom 30 then: @@ -26,102 +26,100 @@ !! \param verb_in Verbosity level. !! function gpmd_fortran(nats,nTypes,coords_in,latticeVectors_in,atomTypes_in,atomicNumbers_in,& - &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) result(err) bind(c, name='gpmd_compute') - - use iso_c_binding, only: c_char, c_double, c_int, c_bool - use gpmdcov_vars - use gpmdcov_lib_mod - - implicit none - integer(c_int), intent(in), value :: nats - integer(c_int), intent(in), value :: nTypes - real(c_double), intent(inout) :: coords_in(3*nats) - real(c_double), intent(inout) :: field_in(3) - real(c_double), intent(inout) :: forces_out(3*nats) - real(c_double), intent(inout) :: charges_out(nats) - real(c_double), intent(inout) :: dipole_out(3), energy_out(1) - integer(c_int), intent(inout) :: atomTypes_in(nats) - integer(c_int) ,intent(inout) :: atomicNumbers_in(nTypes) - real(c_double) ,intent(inout) :: latticeVectors_in(9) - integer(c_int), intent(in), value :: verb_in - logical(c_bool) :: err - - real(dp), allocatable :: coords(:,:) - real(dp), allocatable :: forces(:,:), charges(:), dipole(:) - !real(dp), allocatable :: bornch(:,:) - real(dp), allocatable :: field(:) - real(dp), allocatable :: latticeVectors(:,:) - integer, allocatable :: atomTypes(:), atomicNumbers(:) - integer :: k - integer :: verb - - err = .true. - allocate(coords(3,nats)) - allocate(atomTypes(nats)) - allocate(atomicNumbers(nTypes)) - allocate(latticeVectors(3,3)) - allocate(charges(nats)) - allocate(forces(3,nats)) - allocate(dipole(3)) - allocate(field(3)) - - !Note that arrays appear in another order. We need to rearange - !the data. This is because of the column mayor (in python) vs. - !row mayor in fortran. - do k = 1, nats - coords(1,k) = coords_in((k-1)*3 + 1) - coords(2,k) = coords_in((k-1)*3 + 2) - coords(3,k) = coords_in((k-1)*3 + 3) - enddo - - latticeVectors(1,1) = latticeVectors_in(1) - latticeVectors(1,2) = latticeVectors_in(2) - latticeVectors(1,3) = latticeVectors_in(3) - - latticeVectors(2,1) = latticeVectors_in(4) - latticeVectors(2,2) = latticeVectors_in(5) - latticeVectors(2,3) = latticeVectors_in(6) - - latticeVectors(3,1) = latticeVectors_in(7) - latticeVectors(3,2) = latticeVectors_in(8) - latticeVectors(3,3) = latticeVectors_in(9) - - atomicNumbers = atomicNumbers_in - - do k = 1,nats !We correct for the indexing - atomTypes(k) = atomTypes_in(k) + 1 - enddo - - field = field_in - - verb = verb_in - call gpmd_compute(coords,atomTypes,atomicNumbers,latticeVectors,& - &field,charges,forces,dipole,energy_out,verb) - - write(*,*)"energy_out",energy_out - err = .false. - - !We vectorize/flatten the forces to send back to python - do k = 1, nats - forces_out((k-1)*3 + 1) = forces(1,k) - forces_out((k-1)*3 + 2) = forces(2,k) - forces_out((k-1)*3 + 3) = forces(3,k) - enddo - - charges_out(:) = charges(:) - dipole_out(:) = dipole(:) - - deallocate(coords) - deallocate(forces) - deallocate(charges) - deallocate(latticeVectors) - deallocate(atomTypes) - deallocate(atomicNumbers) - deallocate(dipole) - - return + &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) result(err) bind(c, name='gpmd_compute') + + use iso_c_binding, only: c_char, c_double, c_int, c_bool + use gpmdcov_vars + use gpmdcov_lib_mod + + implicit none + integer(c_int), intent(in), value :: nats + integer(c_int), intent(in), value :: nTypes + real(c_double), intent(inout) :: coords_in(3*nats) + real(c_double), intent(inout) :: field_in(3) + real(c_double), intent(inout) :: forces_out(3*nats) + real(c_double), intent(inout) :: charges_out(nats) + real(c_double), intent(inout) :: dipole_out(3), energy_out(1) + integer(c_int), intent(inout) :: atomTypes_in(nats) + integer(c_int) ,intent(inout) :: atomicNumbers_in(nTypes) + real(c_double) ,intent(inout) :: latticeVectors_in(9) + integer(c_int), intent(in), value :: verb_in + logical(c_bool) :: err + + real(dp), allocatable :: coords(:,:) + real(dp), allocatable :: forces(:,:), charges(:), dipole(:) + !real(dp), allocatable :: bornch(:,:) + real(dp), allocatable :: field(:) + real(dp), allocatable :: latticeVectors(:,:) + integer, allocatable :: atomTypes(:), atomicNumbers(:) + integer :: k + integer :: verb + + err = .true. + allocate(coords(3,nats)) + allocate(atomTypes(nats)) + allocate(atomicNumbers(nTypes)) + allocate(latticeVectors(3,3)) + allocate(charges(nats)) + allocate(forces(3,nats)) + allocate(dipole(3)) + allocate(field(3)) + + !Note that arrays appear in another order. We need to rearange + !the data. This is because of the column mayor (in python) vs. + !row mayor in fortran. + do k = 1, nats + coords(1,k) = coords_in((k-1)*3 + 1) + coords(2,k) = coords_in((k-1)*3 + 2) + coords(3,k) = coords_in((k-1)*3 + 3) + enddo + + latticeVectors(1,1) = latticeVectors_in(1) + latticeVectors(1,2) = latticeVectors_in(2) + latticeVectors(1,3) = latticeVectors_in(3) + + latticeVectors(2,1) = latticeVectors_in(4) + latticeVectors(2,2) = latticeVectors_in(5) + latticeVectors(2,3) = latticeVectors_in(6) + + latticeVectors(3,1) = latticeVectors_in(7) + latticeVectors(3,2) = latticeVectors_in(8) + latticeVectors(3,3) = latticeVectors_in(9) + + atomicNumbers = atomicNumbers_in + + do k = 1,nats !We correct for the indexing + atomTypes(k) = atomTypes_in(k) + 1 + enddo + + field = field_in + + verb = verb_in + call gpmd_compute(coords,atomTypes,atomicNumbers,latticeVectors,& + &field,charges,forces,dipole,energy_out,verb) + + write(*,*)"energy_out",energy_out + + !We vectorize/flatten the forces to send back to python + do k = 1, nats + forces_out((k-1)*3 + 1) = forces(1,k) + forces_out((k-1)*3 + 2) = forces(2,k) + forces_out((k-1)*3 + 3) = forces(3,k) + enddo + + charges_out(:) = charges(:) + dipole_out(:) = dipole(:) + + deallocate(coords) + deallocate(forces) + deallocate(charges) + deallocate(latticeVectors) + deallocate(atomTypes) + deallocate(atomicNumbers) + deallocate(dipole) + + err = err_status + + return end function gpmd_fortran - - - diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 index 6852d4f2..35687baa 100644 --- a/examples/gpmdk/src/gpmdcov_lib_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -4,215 +4,220 @@ module gpmdcov_lib_mod contains -!> High-level program to perform GRAPH-BASED QMD. LIB MODULE -!! -!! \ingroup PROGRAMS -!! \brief High-level program to perform GRAPH-BASED QMD with a DFTB Hamiltonian -!! using a full parallel Graph-based approach. -!! \note To test this program with the \verbatim run_test \endverbatim script. -!! -!! \author C. F. A. Negre -!! (cnegre@lanl.gov) -!! -!! \author S. Mniszewski -!! (smn@lanl.gov) -!! -!! \author A. M. N. Niklasson -!! (amn@lanl.gov) -!! -!! \author M. E. Wall -!! (mewall@lanl.gov) -!! -!! Verbose levels: -!! >= 0- Only print tags. -!! >= 1- Print useful scalar physical data (e.g., Total Energy) -!! >= 2- Print single file output data (e.g., 0-SCF Charges) -!! >= 3- Write out trajectory data. -!! >= 4- Write out physically meaningful 1D arrays (e.g., Charges for the species) -!! >= 5- Write out physically meaningful 2D arrays (e.g., H matrix) -!! -!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& -! &forces_out,charges_out,verb_in) -!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& -! &forces_out,charges_out,verb_in) -!subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& -! &charges_out,forces_out,verb_in) -subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& - &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) - - - ! Local modules - use gpmdcov_vars - use gpmdcov_dm_min_mod - use gpmdcov_energandforces_mod - use gpmdcov_prepareMD_mod - use gpmdcov_mdloop_mod - use gpmdcov_init_mod - use gpmdcov_part_mod - use gpmdcov_assert_mod - use gpmdcov_mod - use gpmdcov_diagonalize_mod - use gpmdcov_kernel_mod - use gpmdcov_highorder_mod - use prg_ptable_mod - use prg_response_mod - - ! Latte modules -#ifdef USE_LATTE - use gpmdcov_latte_mod - use setuparray - use prg_system_mod - use latteparser - use constants_mod - use bml - use nonoarray - use NEBLISTARRAY + !> High-level program to perform GRAPH-BASED QMD. LIB MODULE + !! + !! \ingroup PROGRAMS + !! \brief High-level program to perform GRAPH-BASED QMD with a DFTB Hamiltonian + !! using a full parallel Graph-based approach. + !! \note To test this program with the \verbatim run_test \endverbatim script. + !! + !! \author C. F. A. Negre + !! (cnegre@lanl.gov) + !! + !! \author S. Mniszewski + !! (smn@lanl.gov) + !! + !! \author A. M. N. Niklasson + !! (amn@lanl.gov) + !! + !! \author M. E. Wall + !! (mewall@lanl.gov) + !! + !! Verbose levels: + !! >= 0- Only print tags. + !! >= 1- Print useful scalar physical data (e.g., Total Energy) + !! >= 2- Print single file output data (e.g., 0-SCF Charges) + !! >= 3- Write out trajectory data. + !! >= 4- Write out physically meaningful 1D arrays (e.g., Charges for the species) + !! >= 5- Write out physically meaningful 2D arrays (e.g., H matrix) + !! + !subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + ! &forces_out,charges_out,verb_in) + !subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + ! &forces_out,charges_out,verb_in) + !subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + ! &charges_out,forces_out,verb_in) + subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors_in,& + &field_in,charges_out,forces_out,dipole_out,energy_out,verb_in) + + + ! Local modules + use gpmdcov_vars + use gpmdcov_dm_min_mod + use gpmdcov_energandforces_mod + use gpmdcov_prepareMD_mod + use gpmdcov_mdloop_mod + use gpmdcov_init_mod + use gpmdcov_part_mod + use gpmdcov_assert_mod + use gpmdcov_mod + use gpmdcov_diagonalize_mod + use gpmdcov_kernel_mod + use gpmdcov_highorder_mod + use prg_ptable_mod + use prg_response_mod + + ! Latte modules +#ifdef USE_LATTE + use gpmdcov_latte_mod + use setuparray + use prg_system_mod + use latteparser + use constants_mod + use bml + use nonoarray + use NEBLISTARRAY #endif - implicit none - real(dp), allocatable, intent(in) :: coords_in(:,:), lattice_vectors_in(:,:) - real(dp), allocatable, intent(in) :: field_in(:) - real(dp), allocatable, intent(inout) :: charges_out(:) - real(dp), allocatable, intent(inout) :: forces_out(:,:) - !real(dp), allocatable, intent(inout) :: bornch_out(:,:) - real(dp), allocatable, intent(inout) :: dipole_out(:) - integer, allocatable, intent(in) :: atomTypes_in(:) - real(dp) :: energy_out(1) - integer, intent(in) :: verb_in - integer :: k - integer, allocatable, intent(in) :: atomic_numbers_in(:) - - real(dp),allocatable :: dipoleMoment(:),bornCharges(:,:) - real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) - real(dp) :: factor,dr,dfx,dfy,dfz - integer :: atomI - - !integer :: ipreMD + implicit none + real(dp), allocatable, intent(in) :: coords_in(:,:), lattice_vectors_in(:,:) + real(dp), allocatable, intent(in) :: field_in(:) + real(dp), allocatable, intent(inout) :: charges_out(:) + real(dp), allocatable, intent(inout) :: forces_out(:,:) + !real(dp), allocatable, intent(inout) :: bornch_out(:,:) + real(dp), allocatable, intent(inout) :: dipole_out(:) + integer, allocatable, intent(in) :: atomTypes_in(:) + real(dp) :: energy_out(1) + integer, intent(in) :: verb_in + integer :: k + integer, allocatable, intent(in) :: atomic_numbers_in(:) + + real(dp),allocatable :: dipoleMoment(:),bornCharges(:,:) + real(dp),allocatable :: savedDipoleMoment(:),savedCoords(:,:) + real(dp) :: factor,dr,dfx,dfy,dfz + integer :: atomI + + !integer :: ipreMD !!!!!!!!!!!!!!!!!!!!!!!! - !> Main program driver + !> Main program driver !!!!!!!!!!!!!!!!!!!!!!!! - if(verb_in < 0)then - open(unit=6, file="/dev/null", form="formatted") - endif - - !> Initialize the program variables and parse input files. - sy%nats = size(atomTypes_in) - allocate(sy%symbol(sy%nats)) - allocate(sy%atomic_number(sy%nats)) - allocate(sy%coordinate(3,sy%nats)) - allocate(sy%mass(sy%nats)) - allocate(sy%lattice_vector(3,3)) - - sy%coordinate = coords_in - sy%lattice_vector = lattice_vectors_in - - !In this code symbols, mass, and atomic number are large - !arrays running through all the atoms, not only atom types. - !In gpmd, atomTypes are stored in sp%spindex - allocate(sy%spindex(sy%nats)) - write(*,*)"atom types",atomTypes_in - do k = 1,sy%nats - kk = atomTypes_in(k) - sy%spindex(k) = kk - sy%mass(k) = element_mass(atomic_numbers_in(kk)) - sy%symbol(k) = element_symbol(atomic_numbers_in(kk)) - sy%atomic_number(k) = atomic_numbers_in(kk) - enddo - - sy%nsp = size(atomic_numbers_in) - allocate(sy%splist(sy%nsp)) - allocate(sy%spatnum(sy%nsp)) - allocate(sy%spmass(sy%nsp)) - - do k = 1,sy%nsp - sy%splist(k) = element_symbol(atomic_numbers_in(k)) - sy%spatnum(k) = atomic_numbers_in(k) - sy%spmass(k) = element_mass(atomic_numbers_in(k)) - enddo - write(*,*)"spindex",sy%spindex,"splist",sy%splist - - !deallocate(coords_in, attice_vectors_in) - !deallocate(atomTypes_in) - - write(*,*)"Bef init" - call gpmdcov_Init(.true.) - - lt%stopat = "gpmdcov_Energ" - lt%verbose = verb_in + + lib_mode = .true. + err_status = .false. + + if(verb_in < 0)then + open(unit=6, file="/dev/null", form="formatted") + endif + + !> Initialize the program variables and parse input files. + sy%nats = size(atomTypes_in) + allocate(sy%symbol(sy%nats)) + allocate(sy%atomic_number(sy%nats)) + allocate(sy%coordinate(3,sy%nats)) + allocate(sy%mass(sy%nats)) + allocate(sy%lattice_vector(3,3)) + + sy%coordinate = coords_in + sy%lattice_vector = lattice_vectors_in + + !In this code symbols, mass, and atomic number are large + !arrays running through all the atoms, not only atom types. + !In gpmd, atomTypes are stored in sp%spindex + allocate(sy%spindex(sy%nats)) + write(*,*)"atom types",atomTypes_in + do k = 1,sy%nats + kk = atomTypes_in(k) + sy%spindex(k) = kk + sy%mass(k) = element_mass(atomic_numbers_in(kk)) + sy%symbol(k) = element_symbol(atomic_numbers_in(kk)) + sy%atomic_number(k) = atomic_numbers_in(kk) + enddo + + sy%nsp = size(atomic_numbers_in) + allocate(sy%splist(sy%nsp)) + allocate(sy%spatnum(sy%nsp)) + allocate(sy%spmass(sy%nsp)) + + do k = 1,sy%nsp + sy%splist(k) = element_symbol(atomic_numbers_in(k)) + sy%spatnum(k) = atomic_numbers_in(k) + sy%spmass(k) = element_mass(atomic_numbers_in(k)) + enddo + write(*,*)"spindex",sy%spindex,"splist",sy%splist + + !deallocate(coords_in, attice_vectors_in) + !deallocate(atomTypes_in) + + write(*,*)"Bef init" + call gpmdcov_Init(.true.) + + lt%stopat = "gpmdcov_Energ" + lt%verbose = verb_in #ifdef USE_LATTE - call init_latte() + call init_latte() #endif - write(*,*)"Bef asset" - call gpmdcov_assert_input(myRank) - - !We give a first guess of the Fermi level. - Ef = lt%efermi - - !The inverse of the electronic temperature. - beta = 1.0_dp/lt%kbt - - !> Initial partition of the system based on the covalency graph. - call gpmdcov_Part(1) - if(lt%stopAt == "gpmdcov_Part") stop - - !> Initialize partitions. - - if(lt%method == "DiagEfFull") eig = .false. - - call gpmdcov_InitParts() - - if(norm2(field_in) < 1.0D-10)then - if(.not. eig) then + write(*,*)"Bef asset" + call gpmdcov_assert_input(myRank) + + !We give a first guess of the Fermi level. + Ef = lt%efermi + + !The inverse of the electronic temperature. + beta = 1.0_dp/lt%kbt + + !> Initial partition of the system based on the covalency graph. + call gpmdcov_Part(1) + if(lt%stopAt == "gpmdcov_Part") stop + + !> Initialize partitions. + + if(lt%method == "DiagEfFull") eig = .false. + + call gpmdcov_InitParts() + if(err_status)return + + if(norm2(field_in) < 1.0D-10)then + if(.not. eig) then call gpmdcov_Diagonalize_H0() if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then - call gpmdcov_muFromParts() + call gpmdcov_muFromParts() endif - endif - call gpmdcov_FirstCharges(eig) + endif + call gpmdcov_FirstCharges(eig) - !> First SCF loop up to maxscf. - if(eig)then + !> First SCF loop up to maxscf. + if(eig)then call gpmdcov_DM_Min(lt%maxscf,sy%net_charge,.true.) - else + else call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) - endif - - !> First calculation of energies and forces. - call gpmdcov_EnergAndForces(sy%net_charge) - !stop - - else - call gpmdcov_get_field_forces(tb%norbi,atomic_numbers_in,field_in) - endif - - charges_out(:) = sy%net_charge(:) - forces_out(:,:) = sy%force(:,:) - energy_out(1) = Epot - - if(allocated(dipoleMoment)) deallocate(dipoleMoment) - allocate(dipoleMoment(3)) - factor = 1.0_dp - call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) - dipole_out(:) = dipoleMoment(:) - - if(lt%stopAt == "gpmdcov_Energ") then - lib_init = .true. - call gpmdcov_Finalize() - return - endif - - !> Setup the Molecular Dynamics (MD) calculation. - call gpmdcov_PrepareMD(gpmdt%temp0) - if(lt%stopAt == "gpmdcov_PrepareMD") return - - !> Perform the MD simulation. - call gpmdcov_MDloop() - if(lt%stopAt == "gpmdcov_MDloop")then - call gpmdcov_Finalize() - return - endif + endif + + !> First calculation of energies and forces. + call gpmdcov_EnergAndForces(sy%net_charge) + !stop + + else + call gpmdcov_get_field_forces(tb%norbi,atomic_numbers_in,field_in) + endif + + charges_out(:) = sy%net_charge(:) + forces_out(:,:) = sy%force(:,:) + energy_out(1) = Epot + + if(allocated(dipoleMoment)) deallocate(dipoleMoment) + allocate(dipoleMoment(3)) + factor = 1.0_dp + call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) + dipole_out(:) = dipoleMoment(:) + + if(lt%stopAt == "gpmdcov_Energ") then + lib_init = .true. + call gpmdcov_Finalize() + return + endif + + !> Setup the Molecular Dynamics (MD) calculation. + call gpmdcov_PrepareMD(gpmdt%temp0) + if(lt%stopAt == "gpmdcov_PrepareMD") return + + !> Perform the MD simulation. + call gpmdcov_MDloop() + if(lt%stopAt == "gpmdcov_MDloop")then + call gpmdcov_Finalize() + return + endif end subroutine gpmd_compute diff --git a/examples/gpmdk/src/gpmdcov_mod.F90 b/examples/gpmdk/src/gpmdcov_mod.F90 index 784aa21e..b6231cb4 100644 --- a/examples/gpmdk/src/gpmdcov_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_mod.F90 @@ -27,7 +27,7 @@ subroutine gpmdcov_fermifunction(beta,energies,mu,fermi) real(dp), allocatable :: exparg(:) if(.not.allocated(exparg))then - allocate(exparg(size(energies))) + allocate(exparg(size(energies))) endif biggest = log(HUGE(biggest))-1.0_dp @@ -36,45 +36,45 @@ subroutine gpmdcov_fermifunction(beta,energies,mu,fermi) fermi = 1.0_dp/(exp(exparg) + 1.0_dp) end subroutine gpmdcov_fermifunction - + subroutine gpmdcov_muDyn(nguess,Nr_SCF) use gpmdcov_vars use gpmdcov_writeout_mod real(dp), allocatable :: nguess(:) integer, intent(in) :: Nr_SCF - !Actualize the Fermi level dynamically + !Actualize the Fermi level dynamically - if(.not.allocated(acceprat))then - allocate(acceprat(2)) - acceprat = 0 - Efstep = 0.1_dp - endif + if(.not.allocated(acceprat))then + allocate(acceprat(2)) + acceprat = 0 + Efstep = 0.1_dp + endif - acceprat(2) = acceprat(1) - acceprat(1) = sign(1.0_dp,tch) + acceprat(2) = acceprat(1) + acceprat(1) = sign(1.0_dp,tch) - if(acceprat(2)*acceprat(1) < 0)then - Efstep = Efstep*0.8_dp - else - Efstep = Efstep*1.01_dp - endif + if(acceprat(2)*acceprat(1) < 0)then + Efstep = Efstep*0.8_dp + else + Efstep = Efstep*1.01_dp + endif - if(Nr_SCF.gt.10)then - if(iscf.gt.10)Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) - else - Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) - endif + if(Nr_SCF.gt.10)then + if(iscf.gt.10)Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) + else + Ef = Ef - sign(1.0_dp,tch)*min(tch**2,Efstep) + endif - !Normalize charges to tch - nguess(:) = nguess(:) - tch/real(sy%nats) + !Normalize charges to tch + nguess(:) = nguess(:) - tch/real(sy%nats) - call gpmdcov_msI("gpmdcov_muDyn","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_muDyn","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) end subroutine gpmdcov_muDyn subroutine gpmdcov_muFromParts() - + use gpmdcov_vars use gpmdcov_writeout_mod integer :: mycount @@ -101,309 +101,317 @@ subroutine gpmdcov_muFromParts() do iptt=1,partsInEachRank(myRank) ipt= reshuffle(iptt,myRank) #else - do iptt = 1,gpat%TotalParts - ipt = iptt + do iptt = 1,gpat%TotalParts + ipt = iptt #endif - do i = 1, norbsInEachCHAtRank(iptt) - evalsInRank(i+shift) = syprt(ipt)%estr%evals(i) - dvalsInRank(i+shift) = syprt(ipt)%estr%dvals(i) - mycount = mycount + 1 + do i = 1, norbsInEachCHAtRank(iptt) + evalsInRank(i+shift) = syprt(ipt)%estr%evals(i) + dvalsInRank(i+shift) = syprt(ipt)%estr%dvals(i) + mycount = mycount + 1 + enddo + shift = shift + norbsInEachCHAtRank(iptt) enddo - shift = shift + norbsInEachCHAtRank(iptt) - enddo - -! write(*,*)"norbsInRank",norbsInRank -! write(*,*)"evalsInEachCHAtRank",norbsInEachCHAtRank - - if(.not.allocated(norbsInEachCH)) allocate(norbsInEachCH(gpat%TotalParts)) - norbsInEachCH = 0 - nRanks = getNRanks() - if(.not.allocated(npartsVect))then - allocate(npartsVect(nRanks)) - allocate(displ(nRanks)) - allocate(norbsInEachRank(nRanks)) - endif - do i = 1,nRanks - npartsVect(i) = partsInEachRank(i) - if(i == 1)then - shift = 0 - else - shift = shift + partsInEachRank(i-1) + ! write(*,*)"norbsInRank",norbsInRank + ! write(*,*)"evalsInEachCHAtRank",norbsInEachCHAtRank + + if(.not.allocated(norbsInEachCH)) allocate(norbsInEachCH(gpat%TotalParts)) + norbsInEachCH = 0 + nRanks = getNRanks() + if(.not.allocated(npartsVect))then + allocate(npartsVect(nRanks)) + allocate(displ(nRanks)) + allocate(norbsInEachRank(nRanks)) endif - displ(i) = shift - enddo + + do i = 1,nRanks + npartsVect(i) = partsInEachRank(i) + if(i == 1)then + shift = 0 + else + shift = shift + partsInEachRank(i-1) + endif + displ(i) = shift + enddo #ifdef DO_MPI - call allGatherVIntParallel(norbsInEachCHAtRank, partsInEachRank(myRank),norbsInEachCH ,npartsVect, displ) + call allGatherVIntParallel(norbsInEachCHAtRank, partsInEachRank(myRank),norbsInEachCH ,npartsVect, displ) #else - norbsInEachCH = norbsInEachCHAtRank + norbsInEachCH = norbsInEachCHAtRank #endif - kk = 0 + kk = 0 - norbsInEachRank = 0.0_dp - do i = 1,nRanks - do j = 1,partsInEachRank(i) - kk = kk + 1 - norbsInEachRank(i) = norbsInEachRank(i) + norbsInEachCH(kk) + norbsInEachRank = 0.0_dp + do i = 1,nRanks + do j = 1,partsInEachRank(i) + kk = kk + 1 + norbsInEachRank(i) = norbsInEachRank(i) + norbsInEachCH(kk) + enddo + if(i == 1)then + shift = 0 + else + shift = shift + norbsInEachRank(i-1) + endif + displ(i) = shift enddo - if(i == 1)then - shift = 0 - else - shift = shift + norbsInEachRank(i-1) - endif - displ(i) = shift - enddo - totalNorbs = sum(norbsInEachRank) + totalNorbs = sum(norbsInEachRank) - if(allocated(evalsAll))then - deallocate(evalsAll) - deallocate(dvalsAll) - endif - allocate(evalsAll(totalNorbs)) - allocate(dvalsAll(totalNorbs)) - evalsAll = 0.0_dp - dvalsAll = 0.0_dp + if(allocated(evalsAll))then + deallocate(evalsAll) + deallocate(dvalsAll) + endif + allocate(evalsAll(totalNorbs)) + allocate(dvalsAll(totalNorbs)) + evalsAll = 0.0_dp + dvalsAll = 0.0_dp #ifdef DO_MPI - call allGatherVRealParallel(evalsInRank, norbsInRank, evalsAll ,norbsInEachRank, displ) - call allGatherVRealParallel(dvalsInRank, norbsInRank, dvalsAll ,norbsInEachRank, displ) + call allGatherVRealParallel(evalsInRank, norbsInRank, evalsAll ,norbsInEachRank, displ) + call allGatherVRealParallel(dvalsInRank, norbsInRank, dvalsAll ,norbsInEachRank, displ) #else - evalsAll = evalsInRank - dvalsAll = dvalsInRank + evalsAll = evalsInRank + dvalsAll = dvalsInRank #endif - - nocc = bndfilTotal*real(sy%estr%norbs,dp) - - err = .false. - - mls_mu = mls() - call gpmdcov_musearch(evalsAll,dvalsAll,beta,nocc,10000,10d-10,& - &Ef,HOMO,LUMO,err,myRank,egap_glob,lt%verbose) - if(err .eqv. .true.)then - call gpmdcov_musearch_bisec(evalsAll,dvalsAll,beta,nocc,10d-10,Ef,myRank,egap_glob,lt%verbose) - endif - call gpmdcov_msI("gpmdcov_muFromParts","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) - call gpmdcov_msI("gpmdcov_muFromParts","Time for mu search="//to_string(mls() - mls_mu),lt%verbose,myRank) - - end subroutine gpmdcov_muFromParts - - !> Perform musearch. - !! \param evals Eigenvalues of the system. - !! \param dvals Partial traces. - !! \prarm beta Electronic temperature. - !! \param maxMuIter Maximum numner of iterations. - !! \param occTol Occupation tolerance. - !! \param mu Chemical potential. - !! - subroutine gpmdcov_musearch(evals, dvals, beta, nocc, maxMuIter, occTol, mu, & - & HOMO, LUMO, err, rank, egap, verbose) - use gpmdcov_writeout_mod - implicit none - integer :: i, j, norbs - integer, intent(in) :: maxMuIter, rank - real(dp) :: den, occErr, mu0, occ, muMax,muMin - real(dp), allocatable :: fvals(:) - real(dp), intent(in) :: evals(:), dvals(:) - real(dp), intent(in) :: occTol, beta, nocc - real(dp), intent(inout) :: egap - real(dp), intent(inout) :: mu, HOMO, LUMO - logical, intent(inout) :: err - integer, optional, intent(in) :: verbose - - mu0 = 0.5d0*(evals(int(floor(Nocc))) + evals(int(floor(Nocc))+1)) - muMin = minval(evals) - muMax = maxval(evals) - if (present(verbose)) then - call gpmdcov_msI("gpmdcov_musearch","In gpmdcov_musearch with muMin,muMax,mu0 & - &= "//to_string(muMin)//","//to_string(muMax)//","//to_string(mu0),verbose,rank) - endif - norbs = size(evals, dim = 1) + nocc = bndfilTotal*real(sy%estr%norbs,dp) - if(.not.allocated(fvals))then - allocate(fvals(norbs)) - endif + err = .false. - ! if(mu > (muMin - muMax)/2.0_dp)then - ! mu = (muMin - muMax)/2.0_dp - ! else - ! mu0 = mu !(muMin - muMax)/2.0_dp - ! endif + mls_mu = mls() + call gpmdcov_musearch(evalsAll,dvalsAll,beta,nocc,10000,10d-10,& + &Ef,HOMO,LUMO,err,myRank,egap_glob,lt%verbose) + if(err .eqv. .true.)then + call gpmdcov_musearch_bisec(evalsAll,dvalsAll,beta,nocc,10d-10,Ef,myRank,egap_glob,lib_mode,err_status,lt%verbose) + endif - call gpmdcov_fermifunction(beta,evals,mu0,fvals) - - !do j = 1, norbs - ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) - !end do + if(err_status) return + + call gpmdcov_msI("gpmdcov_muFromParts","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_muFromParts","Time for mu search="//to_string(mls() - mls_mu),lt%verbose,myRank) + + end subroutine gpmdcov_muFromParts + + !> Perform musearch. + !! \param evals Eigenvalues of the system. + !! \param dvals Partial traces. + !! \prarm beta Electronic temperature. + !! \param maxMuIter Maximum numner of iterations. + !! \param occTol Occupation tolerance. + !! \param mu Chemical potential. + !! + subroutine gpmdcov_musearch(evals, dvals, beta, nocc, maxMuIter, occTol, mu, & + & HOMO, LUMO, err, rank, egap, verbose) + use gpmdcov_writeout_mod + implicit none + integer :: i, j, norbs + integer, intent(in) :: maxMuIter, rank + real(dp) :: den, occErr, mu0, occ, muMax,muMin + real(dp), allocatable :: fvals(:) + real(dp), intent(in) :: evals(:), dvals(:) + real(dp), intent(in) :: occTol, beta, nocc + real(dp), intent(inout) :: egap + real(dp), intent(inout) :: mu, HOMO, LUMO + logical, intent(inout) :: err + integer, optional, intent(in) :: verbose + + mu0 = 0.5d0*(evals(int(floor(Nocc))) + evals(int(floor(Nocc))+1)) + muMin = minval(evals) + muMax = maxval(evals) + if (present(verbose)) then + call gpmdcov_msI("gpmdcov_musearch","In gpmdcov_musearch with muMin,muMax,mu0 & + &= "//to_string(muMin)//","//to_string(muMax)//","//to_string(mu0),verbose,rank) + endif + norbs = size(evals, dim = 1) - occ = 0.0_dp - do j = 1, norbs - occ = occ + fvals(j)*dvals(j) - end do + if(.not.allocated(fvals))then + allocate(fvals(norbs)) + endif - occErr = abs(occ-nocc) + ! if(mu > (muMin - muMax)/2.0_dp)then + ! mu = (muMin - muMax)/2.0_dp + ! else + ! mu0 = mu !(muMin - muMax)/2.0_dp + ! endif - do i = 1, maxMuIter + call gpmdcov_fermifunction(beta,evals,mu0,fvals) - call gpmdcov_fermifunction(beta,evals,mu0,fvals) !do j = 1, norbs ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) !end do occ = 0.0_dp - den = 0.0_dp do j = 1, norbs occ = occ + fvals(j)*dvals(j) - den = den + beta*fvals(j)*(1.0_dp-fvals(j))*dvals(j) end do - - occErr = abs(occ - nocc) - if (occErr < occTol) then - mu = mu0 - exit - elseif (den > 1.0d-12) then - mu0 = mu0 + (nocc - occ)/den - mu = mu0 - endif - if(mu > muMax .or. mu < muMin)then - err = .true. - mu = 0.0_dp - return - endif - - if (i .eq. maxMuIter) then - write(*,*) "Max mu iters reached for NR. Trying bisection ..." - err = .true. - mu = 0.0_dp - return - else - if(occErr .lt. occTol) exit - end if - - end do - - deallocate(fvals) - - HOMO = muMin - LUMO = muMax - do i = 1,norbs - if(evals(i) .lt. mu)then - HOMO = max(evals(i),HOMO) - elseif(evals(i) .gt. mu)then - LUMO = min(evals(i),LUMO) - endif - enddo + occErr = abs(occ-nocc) + + do i = 1, maxMuIter + + call gpmdcov_fermifunction(beta,evals,mu0,fvals) + !do j = 1, norbs + ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) + !end do + + occ = 0.0_dp + den = 0.0_dp + do j = 1, norbs + occ = occ + fvals(j)*dvals(j) + den = den + beta*fvals(j)*(1.0_dp-fvals(j))*dvals(j) + end do + + occErr = abs(occ - nocc) + if (occErr < occTol) then + mu = mu0 + exit + elseif (den > 1.0d-12) then + mu0 = mu0 + (nocc - occ)/den + mu = mu0 + endif - egap = LUMO-HOMO + if(mu > muMax .or. mu < muMin)then + err = .true. + mu = 0.0_dp + return + endif - end subroutine gpmdcov_musearch + if (i .eq. maxMuIter) then + write(*,*) "Max mu iters reached for NR. Trying bisection ..." + err = .true. + mu = 0.0_dp + return + else + if(occErr .lt. occTol) exit + end if + end do - !> Routine to compute the Fermi level given a set of eigenvalues and a temperature. - !! It applies the Bisection method over the function: - !! \f$ g(\mu) = \sum_k 2 f(\epsilon_k - \mu) - N = 0 \f$ - !! Where \f$ f(\epsilon_k - \mu) = \frac{1}{1+\exp{(\epsilon_k - \mu)/(k_bT)}}\f$. - !! \param evals Eigenvalues of the system (\f$ \{ \epsilon_k \} \f$). - !! \param kbt Temperature times the Boltzmann's constant (\f$ k_bT \f$). - !! \param bndfil Filing factor (\f$ N_{el}/(2*N_{orbs})\f$). - !! \param tol Tolerance for the bisection method. - !! \param Ef Fermi level (\f$ \mu \f$). - !! - subroutine gpmdcov_musearch_bisec(evals,dvals,beta,noc,tol,mu,rank,egap,verbose) - use gpmdcov_writeout_mod - integer :: i, m - integer :: norb - integer, intent(in) :: rank - integer, optional, intent(in) :: verbose - real(dp) :: Ft1, Ft2, Prod, egap - real(dp) :: step, tol, fermi, mumax, muMin, HOMO, LUMO - real(dp), intent(in) :: noc, evals(:), dvals(:), beta - real(dp) :: mu - real(dp), allocatable :: fvals(:) - - if(.not.allocated(fvals))then - allocate(fvals(size(evals))) - endif + deallocate(fvals) - if (present(verbose)) then - call gpmdcov_msI("gpmdcov_musearch_bisec","In gpmdcov_musearch_bisec ...",verbose,rank) - endif + HOMO = muMin + LUMO = muMax + do i = 1,norbs + if(evals(i) .lt. mu)then + HOMO = max(evals(i),HOMO) + elseif(evals(i) .gt. mu)then + LUMO = min(evals(i),LUMO) + endif + enddo - norb = size(evals,dim=1) - muMin=minval(evals) - muMax = maxval(evals) - mu = muMin - step=abs(muMax-muMin) - Ft1=0.0_dp - Ft2=0.0_dp - prod=0.0_dp - - !Sum of the occupations - call gpmdcov_fermifunction(beta,evals,mu,fvals) - - do i=1,norb - fermi = fvals(i) - ft1 = ft1 + 1.0_dp*fermi*dvals(i) - enddo - ft1=ft1-noc - - do m=1,1000001 - if(m.gt.1000000)then - write(*,*)"Bisection method in gpmdcov_musearch_bisec not converging ..." - stop - endif - if(mu > muMax + 1.0_dp .or. mu < muMin - 1.0_dp)then - write(*,*)"Bisection method is diverging" - write(*,*)"muMin=",muMin,"muMax=",muMax - write(*,*)evals - stop + egap = LUMO-HOMO + + end subroutine gpmdcov_musearch + + + !> Routine to compute the Fermi level given a set of eigenvalues and a temperature. + !! It applies the Bisection method over the function: + !! \f$ g(\mu) = \sum_k 2 f(\epsilon_k - \mu) - N = 0 \f$ + !! Where \f$ f(\epsilon_k - \mu) = \frac{1}{1+\exp{(\epsilon_k - \mu)/(k_bT)}}\f$. + !! \param evals Eigenvalues of the system (\f$ \{ \epsilon_k \} \f$). + !! \param kbt Temperature times the Boltzmann's constant (\f$ k_bT \f$). + !! \param bndfil Filing factor (\f$ N_{el}/(2*N_{orbs})\f$). + !! \param tol Tolerance for the bisection method. + !! \param Ef Fermi level (\f$ \mu \f$). + !! + subroutine gpmdcov_musearch_bisec(evals,dvals,beta,noc,tol,mu,rank,egap,lib_mode,err_status,verbose) + use gpmdcov_writeout_mod + integer :: i, m + integer :: norb + integer, intent(in) :: rank + integer, optional, intent(in) :: verbose + real(dp) :: Ft1, Ft2, Prod, egap + real(dp) :: step, tol, fermi, mumax, muMin, HOMO, LUMO + real(dp), intent(in) :: noc, evals(:), dvals(:), beta + real(dp) :: mu + logical :: err_status,lib_mode + real(dp), allocatable :: fvals(:) + + if(.not.allocated(fvals))then + allocate(fvals(size(evals))) endif - if(abs(ft1).lt.tol)then !tolerance control - exit + if (present(verbose)) then + call gpmdcov_msI("gpmdcov_musearch_bisec","In gpmdcov_musearch_bisec ...",verbose,rank) endif - mu = mu + step + norb = size(evals,dim=1) + muMin=minval(evals) + muMax = maxval(evals) + mu = muMin + step=abs(muMax-muMin) + Ft1=0.0_dp + Ft2=0.0_dp + prod=0.0_dp - ft2=0.0_dp - - !New sum of the occupations + !Sum of the occupations call gpmdcov_fermifunction(beta,evals,mu,fvals) + do i=1,norb fermi = fvals(i) - ft2 = ft2 + 1.0_dp*fermi*dvals(i) + ft1 = ft1 + 1.0_dp*fermi*dvals(i) enddo + ft1=ft1-noc + + do m=1,1000001 + if(m.gt.1000000)then + write(*,*)"Bisection method in gpmdcov_musearch_bisec not converging ..." + if(lib_mode)then + err_status = .true. + return + else + stop + endif + endif + if(mu > muMax + 1.0_dp .or. mu < muMin - 1.0_dp)then + write(*,*)"Bisection method is diverging" + write(*,*)"muMin=",muMin,"muMax=",muMax + write(*,*)evals + stop + endif - ft2=ft2-noc + if(abs(ft1).lt.tol)then !tolerance control + exit + endif - !Product to see the change in sign. - prod = ft2*ft1 - if(prod.lt.0)then - mu=mu-step - step=step/2.0_dp !If the root is inside we shorten the step. - else - ft1=ft2 !If not, Ef moves forward. - endif + mu = mu + step - enddo + ft2=0.0_dp - HOMO = muMin - LUMO = muMax - do i = 1,norb - if(evals(i) .lt. mu)then - HOMO = max(evals(i),HOMO) - elseif(evals(i) .gt. mu)then - LUMO = min(evals(i),LUMO) - endif - enddo - - !write(*,*)"EGAP BISEC",LUMO-HOMO,mu - egap = LUMO-HOMO + !New sum of the occupations + call gpmdcov_fermifunction(beta,evals,mu,fvals) + do i=1,norb + fermi = fvals(i) + ft2 = ft2 + 1.0_dp*fermi*dvals(i) + enddo + + ft2=ft2-noc + + !Product to see the change in sign. + prod = ft2*ft1 + if(prod.lt.0)then + mu=mu-step + step=step/2.0_dp !If the root is inside we shorten the step. + else + ft1=ft2 !If not, Ef moves forward. + endif + + enddo + + HOMO = muMin + LUMO = muMax + do i = 1,norb + if(evals(i) .lt. mu)then + HOMO = max(evals(i),HOMO) + elseif(evals(i) .gt. mu)then + LUMO = min(evals(i),LUMO) + endif + enddo + + !write(*,*)"EGAP BISEC",LUMO-HOMO,mu + egap = LUMO-HOMO - end subroutine gpmdcov_musearch_bisec + end subroutine gpmdcov_musearch_bisec -end module gpmdcov_mod + end module gpmdcov_mod diff --git a/examples/gpmdk/src/gpmdcov_vars.F90 b/examples/gpmdk/src/gpmdcov_vars.F90 index 1544f1dc..93695023 100644 --- a/examples/gpmdk/src/gpmdcov_vars.F90 +++ b/examples/gpmdk/src/gpmdcov_vars.F90 @@ -116,7 +116,9 @@ module gpmdcov_vars logical :: nlistSparse = .false. logical :: reactionDone = .false. logical, save :: lib_init = .false. + logical, save :: lib_mode = .false. logical, save :: lib_init2 = .false. + logical, save :: err_status = .false. type(bml_matrix_t) :: ZK1_bml, ZK2_bml, ZK3_bml type(bml_matrix_t) :: ZK4_bml, ZK5_bml, ZK6_bml diff --git a/examples/gpmdk/src/gpmdk.F90 b/examples/gpmdk/src/gpmdk.F90 index 24be5a2d..5b465802 100644 --- a/examples/gpmdk/src/gpmdk.F90 +++ b/examples/gpmdk/src/gpmdk.F90 @@ -87,6 +87,7 @@ program gpmd call gpmdcov_Diagonalize_H0() if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then call gpmdcov_muFromParts() + if(err_status)return endif endif call gpmdcov_FirstCharges(eig) diff --git a/src/prg_genz_mod.F90 b/src/prg_genz_mod.F90 index 984304f1..29707f17 100644 --- a/src/prg_genz_mod.F90 +++ b/src/prg_genz_mod.F90 @@ -170,8 +170,10 @@ end subroutine prg_init_ZSPmat !! \param threshold Threshold value to use, in this case, only in the backtransformation to ellpack format. !! \param mdim Maximun nonzero to use, in this case, only in the backtransformation to ellpack format. !! \param bml_type the bml type we are passing. + !! \param verbose Verbosity controll level. + !! \param err Error handling variable !! - subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose) + subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose,err_out) ! use extras real(dp) :: err_check character(len=*), intent(in) :: bml_type @@ -179,6 +181,8 @@ subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose) integer :: i, j, mdim, norb integer, intent(in) :: mdimin integer, optional, intent(in) :: verbose + logical, optional, intent(inout) :: err_out + logical :: zeroEval real(dp) :: mls_i real(dp) :: invsqrt, threshold real(dp), allocatable :: nono_evals(:), nonotmp(:,:), smat(:,:), umat(:,:) @@ -187,6 +191,7 @@ subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose) type(bml_matrix_t) :: zmat_bml type(bml_matrix_t), intent(inout) :: smat_bml + if(present(verbose))then if(verbose >= 1)then write(*,*)""; write(*,*)"In buildzdiag ..." @@ -228,15 +233,16 @@ subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose) endif call bml_export_to_dense(umat_bml, umat) - nonotmp=0.0_dp + nonotmp = 0.0_dp + zeroEval = .false. !Doing s^-1/2 u^t !$omp parallel do default(none) & - !$omp shared(norb,nono_evals,umat,nonotmp) & + !$omp shared(norb,nono_evals,umat,nonotmp,zeroEval) & !$omp private(i,j,invsqrt) do i = 1, norb - if(nono_evals(i).lt.0.0_dp) stop 'matrix s has a 0 eigenvalue' + if(nono_evals(i).lt.0.0_dp) zeroEval = .true. invsqrt = 1.0_dp/sqrt(nono_evals(i)) do j = 1, norb nonotmp(i,j) = umat(j,i) * invsqrt @@ -255,6 +261,18 @@ subroutine prg_buildZdiag(smat_bml,zmat_bml,threshold,mdimin,bml_type,verbose) call bml_deallocate(umat_bml) call bml_deallocate(nonotmp_bml) + if(zeroEval)then + if(present(verbose))then + if(present(err_out))then + err_out = .true. + else + stop "ERROR at prg_buildZdiag: Overlap matrix is not possitive definite..." + endif + else + stop "ERROR at prg_buildZdiag: Overlap matrix is not possitive definite..." + endif + endif + end subroutine prg_buildZdiag !> Inverse factorization using Niklasson's algorithm. From a4dc3a253a9acf2aa445bc0aa8fe7d078f29bccf Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 6 Sep 2024 17:48:51 -0600 Subject: [PATCH 44/69] Workaround for Cray matmul bug in gpmdk --- examples/gpmdk/src/gpmdcov_dm_min.F90 | 103 ++++++++++++++---------- examples/gpmdk/src/gpmdcov_kernel.F90 | 35 +++++++- examples/gpmdk/src/gpmdcov_neighbor.F90 | 4 +- 3 files changed, 95 insertions(+), 47 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index e7ea770a..6065b86b 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -283,6 +283,7 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) real(dp) :: tch1 real(8) :: mls_v, mls_coul, mls_mu, mls_red, mls_mix, mls_scfIter real(dp), allocatable :: KK0Res(:) + integer :: reclen converged = .false. if(.not.allocated(charges_old))allocate(charges_old(sy%nats)) @@ -465,47 +466,61 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) mls_mix = mls() if(mix)then ! if(myRank == 1) write(*,*)mdstep,"InIf (mix)" - if( kernel%kernelMixing .and. lt%dokernel)then -! if(myRank == 1) write(*,*)mdstep,"InIf ( kernel%kernelMixing .and. lt%dokernel)" - if( scferror < 0.01_dp .or. newPart)then -! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart)" - if(firstKernel .or. kernel%buildAlways)then -! if(myRank == 1) write(*,*)mdstep,"InIf (firstKernel .or. kernel%buildAlways)" - if(kernel%kernelType == "Full")then - call gpmdcov_getKernel(sy%nats) - elseif(kernel%kernelType == "ByBlocks")then - call gpmdcov_getKernel_byBlocks(sy%nats) - elseif(kernel%kernelType == "ByParts")then - call gpmdcov_getKernel_byParts(syprt,syprtk) - else - write(*,*)"The TypeOfKernel is not implemented" - stop - endif - - firstKernel = .false. - newPart = .false. - if((kernel%kernelType == "ByParts" ) .and. (.not. kernel%updateAfterBuild))then - allocate(kernelTimesRes(sy%nats)) -! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart), Applying K" - !CHANGE Check with Anders - !if(mdstep <= xl%minit)then - call gpmdcov_applyKernel(nguess,charges_old,syprtk,kernelTimesRes) - nguess = charges_old - kernelTimesRes - !endif - ! call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) - ! nguess = charges_old - KK0Res - !nguess = nguess - !endif - - deallocate(kernelTimesRes) - elseif(kernel%kernelType == "ByParts" .and. kernel%updateAfterBuild)then - call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) - nguess = charges_old - KK0Res - else - nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) - endif - else - !Update and apply + if( kernel%kernelMixing .and. lt%dokernel)then + ! if(myRank == 1) write(*,*)mdstep,"InIf ( kernel%kernelMixing .and. lt%dokernel)" + if( scferror < 0.01_dp .or. newPart)then + ! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart)" + if(firstKernel .or. kernel%buildAlways)then + ! if(myRank == 1) write(*,*)mdstep,"InIf (firstKernel .or. kernel%buildAlways)" + if(kernel%kernelType == "Full")then + call gpmdcov_getKernel(sy%nats) + elseif(kernel%kernelType == "ByBlocks")then + call gpmdcov_getKernel_byBlocks(sy%nats) + elseif(kernel%kernelType == "ByParts")then + call gpmdcov_getKernel_byParts(syprt,syprtk) + else + write(*,*)"The TypeOfKernel is not implemented" + stop + endif + + firstKernel = .false. + newPart = .false. + if((kernel%kernelType == "ByParts" ) .and. (.not. kernel%updateAfterBuild))then + allocate(kernelTimesRes(sy%nats)) + ! if(myRank == 1) write(*,*)mdstep,"InIf ( scferror < 0.01_dp .or. newPart), Applying K" + !CHANGE Check with Anders + !if(mdstep <= xl%minit)then + call gpmdcov_applyKernel(nguess,charges_old,syprtk,kernelTimesRes) + nguess = charges_old - kernelTimesRes + !endif + ! call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) + ! nguess = charges_old - KK0Res + !nguess = nguess + !endif + + deallocate(kernelTimesRes) + elseif(kernel%kernelType == "ByParts" .and. kernel%updateAfterBuild)then + call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) + nguess = charges_old - KK0Res + ipt= reshuffle(1,myRank) + ! inquire(iolength=reclen)syprtk(ipt)%estr%ker(:,1) + !open(1234,file="kernel_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") + !write(1234)syprtk(ipt)%estr%ker + ! do i=1,size(ker,dim=2) + ! write(1234)syprtk(ipt)%estr%ker(:,i) + ! enddo + !close(1234) + !open(1234,file="kk0res_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") + !write(1234)KK0Res + ! do i=1,size(ker,dim=2) + ! write(1234)syprtk(ipt)%estr%ker(:,i) + ! enddo + !close(1234) + else + nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) + endif + else !if(firstKernel .or. kernel%buildAlways)then + !Update and apply if(myRank == 1) write(*,*)mdstep,"InIf elseOf( scferror < 0.01_dp .or. newPart)" if(kernel%rankNUpdate > 0)then if(kernel%kernelType == "Full")then @@ -514,6 +529,12 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) elseif(kernel%kernelType == "ByParts")then call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) nguess = charges_old - KK0Res +! open(1234,file="rho_updated_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") +! write(1234)syprt(ipt)%estr%rho + ! do i=1,size(ker,dim=2) + ! write(1234)syprtk(ipt)%estr%ker(:,i) + ! enddo +! close(1234) endif else !Just apply the old one if(kernel%kernelType == "Full")then diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 845b5397..68cdf373 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -865,7 +865,14 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re myqnPart(myj) = myqn(myjj) enddo - K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) + K0ResPart(1:natsCore,iptt) = 0.0_dp + do j=1,natsCore + do i=1,size(myqnPart) + K0ResPart(j,iptt) = K0ResPart(j,iptt)+mysyprtk(ipt)%estr%ker(j,i)*(myqnPart(i)-mynPart(i)) + enddo + enddo + + !K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) !Expand K0resPart into K0Res do myj=1,natsCore @@ -1130,7 +1137,19 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re natsCore = gpat%sgraph(ipt)%llsize f = q1(:,iptt) - v_core_i(:,iptt,irank) - ff(:,iptt,irank) = MATMUL(syprtk(ipt)%estr%ker,f(1:natsCore)) +! ff(:,iptt,irank) = 0.0_dp + do j=1,size(syprtk(ipt)%estr%ker,dim=1) + ff(j,iptt,irank)=0.0_dp + do i=1,natsCore + ff(j,iptt,irank) = ff(j,iptt,irank) + syprtk(ipt)%estr%ker(j,i)*f(i) + enddo + enddo +! do i=1,natsCore +! ff(:,iptt,irank) = ff(:,iptt,irank) + syprtk(ipt)%estr%ker(:,i)*f(i) +! c_i(irank) = c_i(irank) + ff(i,iptt,irank)*K0ResPart(i,iptt) +! enddo + +! ff(:,iptt,irank) = MATMUL(syprtk(ipt)%estr%ker,f(1:natsCore)) c_i(irank) = c_i(irank) + DOT_PRODUCT(ff(1:natsCore,iptt,irank),K0ResPart(1:natsCore,iptt)) do myj=1,natsCore myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 @@ -1315,8 +1334,16 @@ subroutine gpmdcov_applyKernel(my_nguess,my_chargesOld,mysyprtk,kernelTimesRes) myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 chargesOldPart(myj) = my_chargesOld(myjj) nguessPart(myj) = my_nguess(myjj) - enddo - kernelTimesResPart = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) + enddo + + kernelTimesResPart = 0.0_dp + do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) + do i=1,size(nguessPart) + kernelTimesResPart(j) = kernelTimesResPart(j)+mysyprtk(ipt)%estr%ker(j,i)*(nguessPart(i)-chargesOldPart(i)) + enddo + enddo + +! kernelTimesResPart = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) !Expand do myj=1,gpat%sgraph(ipt)%llsize diff --git a/examples/gpmdk/src/gpmdcov_neighbor.F90 b/examples/gpmdk/src/gpmdcov_neighbor.F90 index 9246a923..bb7883de 100644 --- a/examples/gpmdk/src/gpmdcov_neighbor.F90 +++ b/examples/gpmdk/src/gpmdcov_neighbor.F90 @@ -677,7 +677,7 @@ subroutine gpmdcov_get_nlist_box_indices(coords,boxOfI,lattice_vectors,nx,ny,nz, iy = 1 + int(floor(abs((coords(2,i) - miny + smallReal)/(rcuty)))) !small box y-index // iz = 1 + int(floor(abs((coords(3,i) - minz + smallReal)/(rcutz)))) !small box z-index // - write(*,*)coords(:,i),ix,iy,iz,(coords(3,i) - minz + smallReal)/rcutz +! write(*,*)coords(:,i),ix,iy,iz,(coords(3,i) - minz + smallReal)/rcutz if(ix > nx .or. ix < 0)then write(*,*)"ix",ix Stop "Error in box index" @@ -705,7 +705,7 @@ subroutine gpmdcov_get_nlist_box_indices(coords,boxOfI,lattice_vectors,nx,ny,nz, totPerBox(ith) = totPerBox(ith) + 1 !How many per box if(totPerBox(ith) > maxInBox) Stop "Exceeding the max in box allowed" inbox(ith,totPerBox(ith)) = i !Who is in box ith - write(*,*)ix,iy,iz,boxOfI(i),coords(:,i) +! write(*,*)ix,iy,iz,boxOfI(i),coords(:,i) enddo From 26f34d8e51081fd2e38f989a99ff452a959b0e72 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 9 Sep 2024 09:08:02 -0600 Subject: [PATCH 45/69] Report max time and rank for dH+dS --- examples/gpmdk/build_gpmd_nvhpc.sh | 2 +- examples/gpmdk/src/gpmdcov_energandforces.F90 | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/examples/gpmdk/build_gpmd_nvhpc.sh b/examples/gpmdk/build_gpmd_nvhpc.sh index cd5f5779..f2afe32d 100755 --- a/examples/gpmdk/build_gpmd_nvhpc.sh +++ b/examples/gpmdk/build_gpmd_nvhpc.sh @@ -3,7 +3,7 @@ rm -rf build_nvhpc mkdir build_nvhpc cd build_nvhpc cmake -DCMAKE_Fortran_COMPILER="mpifort" -DPROGRESS_MPI="yes" -DGPMDK_NVTX="yes" \ --DEXTRA_FCFLAGS="-g -mp -L${NVHPC_ROOT}/cuda/lib64 -lnvToolsExt" \ +-DEXTRA_FCFLAGS="-g -mp -O2 -L${NVHPC_ROOT}/cuda/lib64 -lnvToolsExt" \ -DCMAKE_PREFIX_PATH="$WORKDIR/qmd-progress/install_nvhpc/;$WORKDIR/qmd-progress/bml/install_nvhpc;$WORKDIR/metis-5.1.0/install" ../src/ make diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 6f52c858..6fe12e4b 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -25,6 +25,7 @@ subroutine gpmdcov_EnergAndForces(charges) real(dp) :: smd_test_force(3), smd_test_energy real(dp) :: deltas(3) real(dp) :: delta_h, energy_plus, energy_minus, denergy, differ + type(rankReduceData_t) :: mpimax_in(1), mpimax_out(1) integer :: k logical :: testsmd @@ -110,7 +111,7 @@ subroutine gpmdcov_EnergAndForces(charges) #ifdef USE_NVTX call nvtxStartRange("get_dH_and_dS",2) #endif - + mls_i = mls() if(gpmdt%usevectsk)then call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& @@ -136,6 +137,22 @@ subroutine gpmdcov_EnergAndForces(charges) #ifdef USE_NVTX call nvtxEndRange #endif + mls_i = mls() - mls_i + + mpimax_in(1)%val = mls_i + mpimax_in(1)%rank = myRank + + call maxRankRealParallel(mpimax_in,mpimax_out, 1) + ! At this point, the answer resides on process root + + + call gpmdcov_msI("gpmdcov_EnergAndForces","Max time for dS and dH is "//to_string(mpimax_out(1)%val)//& + &" on Rank "//to_string(mpimax_out(1)%rank),lt%verbose,myRank) + + +#ifdef DO_MPI + call prg_barrierParallel +#endif if(printRank() == 1 .and. lt%verbose >= 10)then call bml_print_matrix("dH0x_bml",dH0x_bml,0,10,0,10) From e2b232c6a4e53c713bd43a22de044feccf74d82c Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 9 Sep 2024 10:11:13 -0600 Subject: [PATCH 46/69] Fix kernel bug. Better matmul fix. o Kernel before rank update was being used. Fix improves SCF error o Fix the matmul equation by specifying index range on lhs --- examples/gpmdk/src/gpmdcov_kernel.F90 | 41 ++++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index 68cdf373..afb14342 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -865,14 +865,14 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re myqnPart(myj) = myqn(myjj) enddo - K0ResPart(1:natsCore,iptt) = 0.0_dp - do j=1,natsCore - do i=1,size(myqnPart) - K0ResPart(j,iptt) = K0ResPart(j,iptt)+mysyprtk(ipt)%estr%ker(j,i)*(myqnPart(i)-mynPart(i)) - enddo - enddo + !K0ResPart(1:natsCore,iptt) = 0.0_dp + !do j=1,natsCore + ! do i=1,size(myqnPart) + ! K0ResPart(j,iptt) = K0ResPart(j,iptt)+mysyprtk(ipt)%estr%ker(j,i)*(myqnPart(i)-mynPart(i)) + ! enddo + !enddo - !K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) + K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) !Expand K0resPart into K0Res do myj=1,natsCore @@ -1138,17 +1138,18 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re f = q1(:,iptt) - v_core_i(:,iptt,irank) ! ff(:,iptt,irank) = 0.0_dp - do j=1,size(syprtk(ipt)%estr%ker,dim=1) - ff(j,iptt,irank)=0.0_dp - do i=1,natsCore - ff(j,iptt,irank) = ff(j,iptt,irank) + syprtk(ipt)%estr%ker(j,i)*f(i) - enddo - enddo + !do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) + ! ff(j,iptt,irank)=0.0_dp + ! do i=1,natsCore + ! ff(j,iptt,irank) = ff(j,iptt,irank) + mysyprtk(ipt)%estr%ker(j,i)*f(i) + ! enddo + !enddo ! do i=1,natsCore ! ff(:,iptt,irank) = ff(:,iptt,irank) + syprtk(ipt)%estr%ker(:,i)*f(i) ! c_i(irank) = c_i(irank) + ff(i,iptt,irank)*K0ResPart(i,iptt) ! enddo + ff(1:size(mysyprtk(ipt)%estr%ker,dim=1),iptt,irank) = MATMUL(mysyprtk(ipt)%estr%ker,f(1:natsCore)) ! ff(:,iptt,irank) = MATMUL(syprtk(ipt)%estr%ker,f(1:natsCore)) c_i(irank) = c_i(irank) + DOT_PRODUCT(ff(1:natsCore,iptt,irank),K0ResPart(1:natsCore,iptt)) do myj=1,natsCore @@ -1336,14 +1337,14 @@ subroutine gpmdcov_applyKernel(my_nguess,my_chargesOld,mysyprtk,kernelTimesRes) nguessPart(myj) = my_nguess(myjj) enddo - kernelTimesResPart = 0.0_dp - do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) - do i=1,size(nguessPart) - kernelTimesResPart(j) = kernelTimesResPart(j)+mysyprtk(ipt)%estr%ker(j,i)*(nguessPart(i)-chargesOldPart(i)) - enddo - enddo + !kernelTimesResPart = 0.0_dp + !do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) + ! do i=1,size(nguessPart) + ! kernelTimesResPart(j) = kernelTimesResPart(j)+mysyprtk(ipt)%estr%ker(j,i)*(nguessPart(i)-chargesOldPart(i)) + ! enddo + !enddo -! kernelTimesResPart = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) + kernelTimesResPart(1:size(mysyprtk(ipt)%estr%ker,dim=1)) = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) !Expand do myj=1,gpat%sgraph(ipt)%llsize From fde467fed2ffcf1bb035b1d4e1b54ee70707d143 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 10 Sep 2024 12:26:49 -0600 Subject: [PATCH 47/69] Clean up comments and begin working on offload of get_dH_or_dS_vect --- examples/gpmdk/src/gpmdcov_dm_min.F90 | 19 - examples/gpmdk/src/gpmdcov_energandforces.F90 | 599 +++++++++++++++++- examples/gpmdk/src/gpmdcov_kernel.F90 | 31 - 3 files changed, 593 insertions(+), 56 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index 6065b86b..44e37c76 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -503,19 +503,6 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) nguess = charges_old - KK0Res ipt= reshuffle(1,myRank) - ! inquire(iolength=reclen)syprtk(ipt)%estr%ker(:,1) - !open(1234,file="kernel_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") - !write(1234)syprtk(ipt)%estr%ker - ! do i=1,size(ker,dim=2) - ! write(1234)syprtk(ipt)%estr%ker(:,i) - ! enddo - !close(1234) - !open(1234,file="kk0res_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") - !write(1234)KK0Res - ! do i=1,size(ker,dim=2) - ! write(1234)syprtk(ipt)%estr%ker(:,i) - ! enddo - !close(1234) else nguess = charges_old - MATMUL(Ker,(nguess-charges_old)) endif @@ -529,12 +516,6 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) elseif(kernel%kernelType == "ByParts")then call gpmdcov_rankN_update_byParts(nguess,charges_old,syprt,syprtk,kernel%rankNUpdate,KK0Res) nguess = charges_old - KK0Res -! open(1234,file="rho_updated_part_"//to_string(ipt)//".bin",form="unformatted",access="stream",action="write",status="unknown") -! write(1234)syprt(ipt)%estr%rho - ! do i=1,size(ker,dim=2) - ! write(1234)syprtk(ipt)%estr%ker(:,i) - ! enddo -! close(1234) endif else !Just apply the old one if(kernel%kernelType == "Full")then diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 6fe12e4b..855f06ea 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -5,8 +5,599 @@ module gpmdcov_EnergAndForces_mod #ifdef USE_NVTX use gpmdcov_nvtx_mod #endif + use prg_parallel_mod + use bml + use ham_latte_mod + use tbparams_latte_mod + private :: get_dH_or_dS_vect_local, get_skblock_vect_local, bondIntegral_vect_local + private + + integer, parameter :: dp = kind(1.0d0) + + public :: gpmdcov_EnergAndForces + contains + !> Function to calculate the bond integral for a given distance + !! and coefficients set. + !! \param dr distance between atoms. + !! \param f parameters (coefficients) for the bond integral. + function bondIntegral_vect_local(dr,f) result(x) + implicit none + real(dp), allocatable :: rmod(:) + real(dp), intent(in) :: dr(:) + real(dp), intent(in) :: f(:,:) + real(dp) :: x(size(dr)) + logical, allocatable :: low_mask(:),mid_mask(:) + integer :: vect_size + + vect_size = size(dr) + + allocate(low_mask(vect_size)) + allocate(mid_mask(vect_size)) + allocate(rmod(vect_size)) + + mid_mask = dr.gt.f(:,7).and.dr.lt.f(:,8) + low_mask = dr.le.f(:,7) + x = 0.0D0 + + ! Calculate low values + rmod = dr - f(:,6) + x = merge(f(:,1)*exp(rmod*(f(:,2) + rmod*(f(:,3) + rmod*(f(:,4) + f(:,5)*rmod)))),0.0D0,low_mask) + + ! Calculate mid values + rmod = dr - f(:,7) + x = merge(f(:,1)*(f(:,9) + rmod*(f(:,10) + rmod*(f(:,11) + rmod*(f(:,12) + rmod*(f(:,13) + rmod*f(:,14)))))),x,mid_mask) + + deallocate(low_mask) + deallocate(mid_mask) + deallocate(rmod) + + end function bondIntegral_vect_local + + !> Standard Slater-Koster sp-parameterization for an atomic block between a pair of atoms + !! \param sp1 Species index for atom 1. This can be obtained from the + !! system type as following: + !! \verbatim sp1 = system%spindex(atom1) \endverbatim + !! \param sp2 Species index for atom 2. + !! \param coorda Coordinates for atom 1. + !! \param coordb Coordinates for atom 2. + !! \param lattice_vectors Lattice vectors for the system. This can be obtained from + !! the system type as following: + !! \verbatim lattice_vectors = system%lattice_vectors \endverbatim + !! \param norbi Number of orbitals for every species in the system. This can be obtained from + !! the tbparams type as following: + !! \verbatim norbi = tbparams%norbi \endverbatim + !! \param onsites Onsites energies for every pair of equal type. Two different variants + !! onsitesH and onsitesS will be used as inputs (see get_hsmat routine) Allocation: + !! \verbatim onsites(maxints,nsp) \endverbatim + !! \param intParams See intpairs_type. + !! \param block Output parameter SK block. + !! \param atnum Input atom number + subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& + ,norbs,onsites,intParams1,intParams2,blk_out,atnum) + implicit none + integer :: dimi, dimj, i, nr_shift_X + integer :: nr_shift_Y, nr_shift_Z, nats, norbsall, mask_size, iorb + integer, intent(in) :: norbs(:), sp(:), atnum + real(dp), allocatable :: HPPP(:), HPPS(:), HSPS(:), HSSS(:), PPSMPP(:) + real(dp), allocatable :: L(:), M(:), N(:) + real(dp), allocatable :: dr(:), dr_m(:), rab(:,:) + real(dp), allocatable :: onsites_m(:) + real(dp), intent(inout) :: blk_out(:,:) + real(dp), allocatable :: blk(:,:) + real(dp), intent(in) :: refcoord(:),coord(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: onsites(:,:) + real(dp), intent(in) :: intParams1(:,:,:),intParams2(:,:,:) + logical, allocatable :: dist_mask(:), onsite_mask(:), calc_mask(:), calcs_mask(:) + logical, allocatable :: calcsp_mask(:), param_mask(:,:), calc_mask_for_porbs(:) + logical, allocatable :: sorb_mask(:), pxorb_mask(:), pyorb_mask(:), pzorb_mask(:) + logical, allocatable :: sorb_at_mask(:), sporb_at_mask(:) + integer, allocatable :: atomidx(:), atomidx_m(:), orbidx(:), orbidx_m(:), orbidx_sel(:) + real(dp), allocatable :: intParams(:,:) + + nats = size(coord,dim=2) + norbsall = sum(norbs) + + allocate(blk(size(blk_out,dim=1),size(blk_out,dim=2))) + + blk(:,:)=0.0_dp + + if(allocated(dr))then + if(nats.ne.size(dr,dim=1))then + deallocate(dr) + deallocate(atomidx) + deallocate(orbidx) + deallocate(rab) + deallocate(dist_mask) + deallocate(onsite_mask) + deallocate(calc_mask) + deallocate(calcs_mask) + deallocate(calcsp_mask) + deallocate(param_mask) + deallocate(sorb_mask) + deallocate(pxorb_mask) + deallocate(pyorb_mask) + deallocate(pzorb_mask) + deallocate(sorb_at_mask) + deallocate(sporb_at_mask) + endif + endif + if(.not.allocated(dr))then + allocate(dr(nats)) + allocate(atomidx(nats)) + atomidx = (/(i,i=1,nats)/) + allocate(orbidx(norbsall)) + orbidx = (/(i,i=1,norbsall)/) + allocate(rab(nats,3)) + allocate(dist_mask(nats)) + allocate(onsite_mask(nats)) + allocate(calc_mask(nats)) + allocate(calcs_mask(nats)) + allocate(calcsp_mask(nats)) + allocate(param_mask(nats,16)) + allocate(sorb_at_mask(nats)) + allocate(sporb_at_mask(nats)) + allocate(sorb_mask(norbsall)) + allocate(pxorb_mask(norbsall)) + allocate(pyorb_mask(norbsall)) + allocate(pzorb_mask(norbsall)) + sorb_at_mask = .false. + sporb_at_mask = .false. + onsite_mask = .false. + sorb_mask = .false. + pxorb_mask = .false. + pyorb_mask = .false. + pzorb_mask = .false. + iorb = 1 + do i = 1,nats + if(i.eq.atnum)then + onsite_mask(i) = .true. + blk(1,iorb) = onsites(1,sp(atnum)) + if(norbs(i).eq.4)then + blk(2,iorb+1) = onsites(2,sp(atnum)) + blk(3,iorb+2) = onsites(3,sp(atnum)) + blk(4,iorb+3) = onsites(4,sp(atnum)) + endif + endif + sorb_mask(iorb) = .true. + iorb = iorb + 1 + if(norbs(i).eq.1)then + sorb_at_mask(i) = .true. + elseif(norbs(i).eq.4)then + sporb_at_mask(i) = .true. + pxorb_mask(iorb) = .true. + pyorb_mask(iorb+1) = .true. + pzorb_mask(iorb+2) = .true. + iorb = iorb + 3 + else + write(*,*)"GET_SKBLOCK_VECT: number of orbitals not 1 or 4 for atom ",i,": ABORT" + stop + endif + enddo + endif + + do i = 1,3 + Rab(:,i) = coord(i,:) + Rab(:,i) = modulo((Rab(:,i) - refcoord(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) + enddo + + dR(:) = norm2(Rab(:,:),dim=2) + + dist_mask = dr.lt.6.5_dp + calc_mask = dist_mask.and..not.onsite_mask + calcsp_mask = calc_mask.and.sporb_at_mask + calcs_mask = calc_mask.and.sorb_at_mask + ! First mask using the s orbitals + ! The s-s term for all atoms is the same in this case + ! The sorb_mask has the same number of .true. elements as the calc_mask + orbidx_m = pack(orbidx,mask=sorb_mask) + orbidx_sel = pack(orbidx_m,mask=calc_mask) + atomidx_m = pack(atomidx,mask=calc_mask) + mask_size = size(atomidx_m) + allocate(intParams(mask_size,16)) + intParams(:,:) = intParams1(atomidx_m(:),:,1) + blk(1,orbidx_sel) = BondIntegral_vect_local(dr(atomidx_m(:)),intParams) + deallocate(intParams) + deallocate(atomidx_m) + deallocate(orbidx_sel) + ! If the atnum atom is a sp atom, calculate the p*-s elements for + ! s orbital atoms and sp orbital atoms separately. Do the s orbital + ! atoms first here. + if(norbs(atnum).eq.4)then + ! First the s orbital atoms + atomidx_m = pack(atomidx,mask=calcs_mask) + mask_size = size(atomidx_m) + allocate(intParams(mask_size,16)) + allocate(HSPS(mask_size)) + intParams(:,:) = intParams1(atomidx_m(:),:,2) + HSPS = BondIntegral_vect_local(dr(atomidx_m(:)),intParams) + orbidx_sel = pack(orbidx_m,mask=calcs_mask) ! orbidx_m still is the s orbital mask + if(size(orbidx_sel).ne.size(atomidx_m))then + write(*,*)"GET_SKBLOCK_VECT: size mismatch of orbital and atom index arrays. Abort." + stop + endif + blk(2,orbidx_sel) = - rab(atomidx_m(:),1)/dr(atomidx_m(:)) * HSPS + blk(3,orbidx_sel) = - rab(atomidx_m(:),2)/dr(atomidx_m(:)) * HSPS + blk(4,orbidx_sel) = - rab(atomidx_m(:),3)/dr(atomidx_m(:)) * HSPS + deallocate(intParams) + deallocate(HSPS) + deallocate(orbidx_sel) + deallocate(atomidx_m) + endif + ! Now work on the sp orbital atoms + ! First create some common masked arrays + atomidx_m = pack(atomidx,mask=calcsp_mask) + mask_size = size(atomidx_m) + dr_m = pack(dr,calcsp_mask) + allocate(L(mask_size)) + allocate(M(mask_size)) + allocate(N(mask_size)) + L = rab(atomidx_m(:),1)/dr_m + M = rab(atomidx_m(:),2)/dr_m + N = rab(atomidx_m(:),3)/dr_m + ! Now work on the p*-s elements first, if relevant + allocate(intParams(mask_size,16)) + allocate(HSPS(mask_size)) + if(norbs(atnum).eq.4)then + intParams(:,:) = intParams2(atomidx_m(:),:,2) + HSPS = BondIntegral_vect_local(dr_m,intParams) + orbidx_sel = pack(orbidx_m,mask=calcsp_mask) ! orbidx_m still is the s orbital mask + if(size(orbidx_sel).ne.size(atomidx_m))then + write(*,*)"GET_SKBLOCK_VECT: size mismatch of orbital and sp atom index arrays. Abort." + stop + endif + blk(2,orbidx_sel) = - L * HSPS + blk(3,orbidx_sel) = - M * HSPS + blk(4,orbidx_sel) = - N * HSPS + deallocate(orbidx_sel) + endif + ! At this point all of the s orbital calculations are done. Move on to p orbitals. + calc_mask_for_porbs = pack(calc_mask,sporb_at_mask) + intParams(:,:) = intParams1(atomidx_m(:),:,2) + HSPS = BondIntegral_vect_local(dr_m,intParams) + if(norbs(atnum).eq.4)then ! Calculate extra params if the reference atom is sp type + allocate(HPPP(mask_size)) + allocate(PPSMPP(mask_size)) + intParams(:,:) = intParams1(atomidx_m(:),:,3) + PPSMPP = BondIntegral_vect_local(dr_m,intParams) + intParams(:,:) = intParams1(atomidx_m(:),:,4) + HPPP = BondIntegral_vect_local(dr_m,intParams) + PPSMPP = PPSMPP - HPPP + endif + deallocate(orbidx_m) + orbidx_m = pack(orbidx,mask=pxorb_mask) ! Select px orbitals + orbidx_sel = pack(orbidx_m,calc_mask_for_porbs) + blk(1,orbidx_sel) = + L(:) * HSPS(:) + if(norbs(atnum).eq.4)then + blk(2,orbidx_sel) = HPPP + L*L*PPSMPP + blk(3,orbidx_sel) = M*L*PPSMPP + blk(4,orbidx_sel) = N*L*PPSMPP + endif + blk(1,orbidx_sel+1) = + M(:) * HSPS(:) + if(norbs(atnum).eq.4)then + blk(2,orbidx_sel+1) = L*M*PPSMPP + blk(3,orbidx_sel+1) = HPPP + M*M*PPSMPP + blk(4,orbidx_sel+1) = N*M*PPSMPP + endif + blk(1,orbidx_sel+2) = + N(:) * HSPS(:) + if(norbs(atnum).eq.4)then + blk(2,orbidx_sel+2) = L*N*PPSMPP + blk(3,orbidx_sel+2) = M*N*PPSMPP + blk(4,orbidx_sel+2) = HPPP + N*N*PPSMPP + deallocate(HPPP) + endif + blk_out(:,:) = blk(:,:) + deallocate(blk) + deallocate(orbidx_m) + deallocatE(orbidx_sel) + deallocate(calc_mask_for_porbs) + deallocate(HSPS) + deallocate(intParams) + deallocate(L) + deallocate(M) + deallocate(N) + deallocate(dr_m) + end subroutine get_SKBlock_vect_local + + !> This routine computes the derivative of H matrix. + !! \param dx X differential to compute the derivatives + !! \param coords System coordinates. + !! \param hindex Contains the Hamiltonian indices for every atom (see get_hindex). + !! \param spindex Species indices (see system_type). + !! \param intPairsH See defprg_inition in intPairs_type + !! \param onsitesH Onsite energies for every orbital of a particular species. + !! \param symbol System element symbol. + !! \param lattice_vectors System lattece vectors. + !! \param norb Number of total orbitals. + !! \param norbi Number of orbitals for each atomic site. + !! \param threshold Threshold value for matrix elements. + !! \param dH0x_bml x derivative of H0. + !! \param dH0y_bml y derivative of H0. + !! \param dH0z_bml z derivative of H0. + !! + subroutine get_dH_or_dS_vect_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol,lattice_vectors, norb, norbi, bml_type, & + threshold, dH0x_bml,dH0y_bml,dH0z_bml) + implicit none + character(2) :: Type_pair(2) + character(2), intent(in) :: symbol(:) + character(len=*), intent(in) :: bml_type + integer :: IDim, JDim, nats, dimi + integer :: dimj, i, ii, j + integer :: jj, l + integer, intent(in) :: hindex(:,:), norb, norbi(:), spindex(:) + integer :: maxnorbi + real(dp) :: Rax_m(3), Rax_p(3), Ray_m(3), Ray_p(3) + real(dp) :: Raz_m(3), Raz_p(3), Rb(3), d, maxblockij + real(dp), allocatable :: Rx(:), Ry(:), Rz(:), blockm(:,:,:) + real(dp), allocatable :: blockp(:,:,:), dh0(:,:), dH0x(:,:), dH0y(:,:), dH0z(:,:) + real(dp), intent(in) :: coords(:,:), dx, lattice_vectors(:,:), onsitesH(:,:) + real(dp), intent(in) :: threshold + type(bml_matrix_t), intent(inout) :: dH0x_bml, dH0y_bml, dH0z_bml + type(intpairs_type), intent(in) :: intPairsH(:,:) + real(dp), allocatable :: intParams1(:,:,:), intParams2(:,:,:) + real(dp), allocatable :: dHx_vect(:,:),dHy_vect(:,:),dHz_vect(:,:),ham_vect(:,:) + integer, allocatable :: norbs_atidx(:) + logical :: test_accuracy + ! integer, intent(in) :: nnstruct(:,:) + + test_accuracy = .false. + + nats = size(coords,dim=2) + + if(bml_get_N(dH0x_bml).LT.0)then + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + else + call bml_deallocate(dH0x_bml) + call bml_deallocate(dH0y_bml) + call bml_deallocate(dH0z_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + endif + + ! dH0x = zeros(HDIM,HDIM); dH0y = zeros(HDIM,HDIM); dH0z = zeros(HDIM,HDIM); + + if(test_accuracy)then + allocate(dH0x(norb,norb)) + allocate(dH0y(norb,norb)) + allocate(dH0z(norb,norb)) + allocate(blockm(maxnorbi,maxnorbi,nats)) + allocate(blockp(maxnorbi,maxnorbi,nats)) + + dH0x = 0.0_dp + dH0y = 0.0_dp + dH0z = 0.0_dp + endif + + allocate(dHx_vect(norb,norb)) + allocate(dHy_vect(norb,norb)) + allocate(dHz_vect(norb,norb)) + allocate(ham_vect(norb,norb)) + + allocate(Rx(nats)) + allocate(Ry(nats)) + allocate(Rz(nats)) + + Rx = coords(1,:) + Ry = coords(2,:) + Rz = coords(3,:) + + maxnorbi = maxval(norbi) + + if(.not.allocated(norbs_atidx))then + allocate(norbs_atidx(nats)) + do i=1,nats + norbs_atidx(i) = norbi(spindex(i)) + enddo + endif + + if(test_accuracy)then + !$omp parallel do default(none) private(i) & + !$omp private(Rax_p,Rax_m,Ray_p,Ray_m,Raz_p,Raz_m) & + !$omp private(dimi,J,Type_pair,dimj,Rb,maxblockij) & + !$omp shared(nats,RX,RY,RZ,spindex,hindex,lattice_vectors, dx, threshold) & + !$omp shared(norbi,intPairsH,onsitesH,symbol,dH0x_bml,dH0y_bml,dH0z_bml) & + !$omp shared(blockm, blockp, dH0x, dH0y, dH0z) + do I = 1, nats + + Type_pair(1) = symbol(i); + Rax_p(1) = RX(I)+ dx; Rax_p(2) = RY(I); Rax_p(3) = RZ(I) + Rax_m(1) = RX(I)- dx; Rax_m(2) = RY(I); Rax_m(3) = RZ(I) + Ray_p(1) = RX(I); Ray_p(2) = RY(I)+dx; Ray_p(3) = RZ(I) + Ray_m(1) = RX(I); Ray_m(2) = RY(I)-dx; Ray_m(3) = RZ(I) + Raz_p(1) = RX(I); Raz_p(2) = RY(I); Raz_p(3) = RZ(I)+dx + Raz_m(1) = RX(I); Raz_m(2) = RY(I); Raz_m(3) = RZ(I)-dx + + dimi = hindex(2,I)-hindex(1,I)+1; + do J = 1,nats + if(J .ne. I)then + + Type_pair(2) = symbol(J); + Rb(1) = RX(J); Rb(2) = RY(J); Rb(3) = RZ(J) + dimj = hindex(2,J)-hindex(1,J)+1; + + !! MATLAB code + ! [fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,Es,Ep,U] = LoadBondIntegralParameters_H(Type_pair); % Used in BondIntegral(dR,fxx_xx) + ! diagonal(1:2) = [Es,Ep]; + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + + call get_SKBlock(spindex(i),spindex(j),Rax_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockp,i) + + maxblockij = 0.0_dp + do ii=1,dimi + do jj=1,dimj + maxblockij = max(maxblockij,abs(blockp(ii,jj,i))) + enddo + enddo + + if(maxblockij.gt.0.0_dp)then + + call get_SKBlock(spindex(i),spindex(j),Rax_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockm,i) + + blockp(:,:,i) = (blockp(:,:,i) - blockm(:,:,i))/(2.0_dp*dx) + + do jj=1,dimj + do ii=1,dimi + dH0x(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blockp(ii,jj,i) + enddo + enddo + + !! MATLAB code + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + + call get_SKBlock(spindex(i),spindex(j),Ray_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockp,i) + + call get_SKBlock(spindex(i),spindex(j),Ray_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockm,i) + + blockp(:,:,i) = (blockp(:,:,i) - blockm(:,:,i))/(2.0_dp*dx) + + do jj=1,dimj + do ii=1,dimi + dH0y(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blockp(ii,jj,i) + enddo + enddo + + !! MATLAB code + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + + call get_SKBlock(spindex(i),spindex(j),Raz_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockp,i) + + call get_SKBlock(spindex(i),spindex(j),Raz_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blockm,i) + + blockp(:,:,i) = (blockp(:,:,i) - blockm(:,:,i))/(2.0_dp*dx) + + do jj=1,dimj + do ii=1,dimi + dH0z(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blockp(ii,jj,i) + enddo + enddo + + endif + endif + enddo + enddo + !$omp end parallel do + endif + + allocate(intParams1(nats,16,4)) + allocate(intParams2(nats,16,4)) + +#ifdef USE_NVTX + call nvtxStartRange("OMP loop",2) +#endif + + !$omp parallel do default(none) private(i) & + !$omp private(Rax_p,Rax_m,Ray_p,Ray_m,Raz_p,Raz_m) & + !$omp private(J) & + !$omp private(intParams1, intParams2) & + !$omp shared(nats,RX,RY,RZ,spindex,hindex,lattice_vectors, dx, threshold) & + !$omp shared(norbs_atidx,intPairsH,onsitesH) & + !$omp shared(coords,ham_vect) & + !$omp shared(dHx_vect, dHy_vect, dHz_vect) + do I = 1, nats + do j = 1,nats + intParams1(j,:,:) = intPairsH(spindex(i),spindex(j))%intParams(:,1:4) + intParams2(j,:,:) = intPairsH(spindex(j),spindex(i))%intParams(:,1:4) + enddo + + Rax_p(1) = RX(I)+ dx; Rax_p(2) = RY(I); Rax_p(3) = RZ(I) + Rax_m(1) = RX(I)- dx; Rax_m(2) = RY(I); Rax_m(3) = RZ(I) + Ray_p(1) = RX(I); Ray_p(2) = RY(I)+dx; Ray_p(3) = RZ(I) + Ray_m(1) = RX(I); Ray_m(2) = RY(I)-dx; Ray_m(3) = RZ(I) + Raz_p(1) = RX(I); Raz_p(2) = RY(I); Raz_p(3) = RZ(I)+dx + Raz_m(1) = RX(I); Raz_m(2) = RY(I); Raz_m(3) = RZ(I)-dx + + call get_SKBlock_vect_local(spindex,rax_p,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),dHx_vect(hindex(1,i):hindex(2,i),:),i) + + call get_SKBlock_vect_local(spindex,rax_m,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),ham_vect(hindex(1,i):hindex(2,i),:),i) + + dHx_vect(hindex(1,i):hindex(2,i),:) = (dHx_vect(hindex(1,i):hindex(2,i),:) & + - ham_vect(hindex(1,i):hindex(2,i),:))/(2.0_dp*dx) + + call get_SKBlock_vect_local(spindex,ray_p,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),dHy_vect(hindex(1,i):hindex(2,i),:),i) + + call get_SKBlock_vect_local(spindex,ray_m,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),ham_vect(hindex(1,i):hindex(2,i),:),i) + + dHy_vect(hindex(1,i):hindex(2,i),:) = (dHy_vect(hindex(1,i):hindex(2,i),:) & + - ham_vect(hindex(1,i):hindex(2,i),:))/(2.0_dp*dx) + + call get_SKBlock_vect_local(spindex,raz_p,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),dHz_vect(hindex(1,i):hindex(2,i),:),i) + + call get_SKBlock_vect_local(spindex,raz_m,coords(:,:),lattice_vectors,norbs_atidx,& + onsitesH,intParams1(:,:,:),intParams2(:,:,:),ham_vect(hindex(1,i):hindex(2,i),:),i) + + dHz_vect(hindex(1,i):hindex(2,i),:) = (dHz_vect(hindex(1,i):hindex(2,i),:) & + - ham_vect(hindex(1,i):hindex(2,i),:))/(2.0_dp*dx) + enddo + + !$omp end parallel do + +#ifdef USE_NVTX + call nvtxEndRange +#endif + + call prg_barrierParallel + + if(test_accuracy)then + if(.not.all(abs(dHx_vect-dH0x).lt.1.D-9))then + do i = 1,norb + do j = 1,norb + if(abs(dHx_vect(i,j)-dH0x(i,j)).ge.1.D-9)then + write(*,*)"GET_DH_OR_DS_VECT: Vectorized dHx differs at i,j = ",i,j,"with difference ",dHx_vect(i,j)-dH0x(i,j) + endif + enddo + enddo + endif + endif + + call bml_import_from_dense(bml_type,dHx_vect,dH0x_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dHy_vect,dH0y_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dHz_vect,dH0z_bml,threshold,norb) !Dense to dense_bml + + if (allocated(dH0x)) then + deallocate(dH0x) + deallocate(dH0y) + deallocate(dH0z) + deallocate(blockm) + deallocate(blockp) + endif + + deallocate(dHx_vect) + deallocate(dHy_vect) + deallocate(dHz_vect) + deallocate(ham_vect) + + ! stop + end subroutine get_dH_or_dS_vect_local subroutine gpmdcov_EnergAndForces(charges) @@ -114,12 +705,12 @@ subroutine gpmdcov_EnergAndForces(charges) mls_i = mls() if(gpmdt%usevectsk)then - call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + call get_dH_or_dS_vect_local(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) - call get_dH_or_dS_vect(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + call get_dH_or_dS_vect_local(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dSx_bml,dSy_bml,dSz_bml) @@ -150,10 +741,6 @@ subroutine gpmdcov_EnergAndForces(charges) &" on Rank "//to_string(mpimax_out(1)%rank),lt%verbose,myRank) -#ifdef DO_MPI - call prg_barrierParallel -#endif - if(printRank() == 1 .and. lt%verbose >= 10)then call bml_print_matrix("dH0x_bml",dH0x_bml,0,10,0,10) call bml_print_matrix("dH0y_bml",dH0y_bml,0,10,0,10) diff --git a/examples/gpmdk/src/gpmdcov_kernel.F90 b/examples/gpmdk/src/gpmdcov_kernel.F90 index afb14342..a162aacd 100644 --- a/examples/gpmdk/src/gpmdcov_kernel.F90 +++ b/examples/gpmdk/src/gpmdcov_kernel.F90 @@ -865,13 +865,6 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re myqnPart(myj) = myqn(myjj) enddo - !K0ResPart(1:natsCore,iptt) = 0.0_dp - !do j=1,natsCore - ! do i=1,size(myqnPart) - ! K0ResPart(j,iptt) = K0ResPart(j,iptt)+mysyprtk(ipt)%estr%ker(j,i)*(myqnPart(i)-mynPart(i)) - ! enddo - !enddo - K0ResPart(1:natsCore,iptt) = MATMUL(mysyprtk(ipt)%estr%ker,(myqnPart-mynPart)) !Expand K0resPart into K0Res @@ -1137,20 +1130,8 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re natsCore = gpat%sgraph(ipt)%llsize f = q1(:,iptt) - v_core_i(:,iptt,irank) -! ff(:,iptt,irank) = 0.0_dp - !do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) - ! ff(j,iptt,irank)=0.0_dp - ! do i=1,natsCore - ! ff(j,iptt,irank) = ff(j,iptt,irank) + mysyprtk(ipt)%estr%ker(j,i)*f(i) - ! enddo - !enddo -! do i=1,natsCore -! ff(:,iptt,irank) = ff(:,iptt,irank) + syprtk(ipt)%estr%ker(:,i)*f(i) -! c_i(irank) = c_i(irank) + ff(i,iptt,irank)*K0ResPart(i,iptt) -! enddo ff(1:size(mysyprtk(ipt)%estr%ker,dim=1),iptt,irank) = MATMUL(mysyprtk(ipt)%estr%ker,f(1:natsCore)) -! ff(:,iptt,irank) = MATMUL(syprtk(ipt)%estr%ker,f(1:natsCore)) c_i(irank) = c_i(irank) + DOT_PRODUCT(ff(1:natsCore,iptt,irank),K0ResPart(1:natsCore,iptt)) do myj=1,natsCore myjj = gpat%sgraph(ipt)%core_halo_index(myj)+1 @@ -1246,11 +1227,6 @@ subroutine gpmdcov_rankN_update_byParts(myqn,myn,mysyprt,mysyprtk,maxRanks,KK0Re error = norm2(IdK0Res - K0Res)/norm2(K0Res) if(myRank == 1)write(*,*)"Error Rank-Update",error,mRanks - !do myi = 1,mRanks - ! do myj = 1,mRanks - ! write(*,*) DOT_PRODUCT(dr_save(:,myi),dr_save(:,myj)) - ! enddo - ! enddo #ifdef USE_NVTX call nvtxEndRange #endif @@ -1337,13 +1313,6 @@ subroutine gpmdcov_applyKernel(my_nguess,my_chargesOld,mysyprtk,kernelTimesRes) nguessPart(myj) = my_nguess(myjj) enddo - !kernelTimesResPart = 0.0_dp - !do j=1,size(mysyprtk(ipt)%estr%ker,dim=1) - ! do i=1,size(nguessPart) - ! kernelTimesResPart(j) = kernelTimesResPart(j)+mysyprtk(ipt)%estr%ker(j,i)*(nguessPart(i)-chargesOldPart(i)) - ! enddo - !enddo - kernelTimesResPart(1:size(mysyprtk(ipt)%estr%ker,dim=1)) = MATMUL(mysyprtk(ipt)%estr%ker,(nguessPart-chargesOldPart)) !Expand From 59a45c5e07464d865e677a627297102b3af12e24 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Tue, 10 Sep 2024 22:35:06 -0600 Subject: [PATCH 48/69] Added protections against double alloc --- examples/gpmdk/src/gpmdcov_lib_mod.F90 | 7 ++++++- src/prg_parallel_mod.F90 | 4 ++-- src/prg_timer_mod.F90 | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 index 35687baa..2d70ddbd 100644 --- a/examples/gpmdk/src/gpmdcov_lib_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -166,7 +166,12 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors if(lt%method == "DiagEfFull") eig = .false. call gpmdcov_InitParts() - if(err_status)return + if(err_status)then + call gpmdcov_Finalize() + call prg_shutdownParallel + call prg_timer_shutdown + return + endif if(norm2(field_in) < 1.0D-10)then if(.not. eig) then diff --git a/src/prg_parallel_mod.F90 b/src/prg_parallel_mod.F90 index bbb4e90d..ed4ecfb7 100644 --- a/src/prg_parallel_mod.F90 +++ b/src/prg_parallel_mod.F90 @@ -152,8 +152,8 @@ end subroutine prg_initParallel ! subroutine prg_shutdownParallel() - deallocate(requestList) - deallocate(rUsed) + if(allocated(requestlist))deallocate(requestList) + if(allocated(rUsed))deallocate(rUsed) #ifdef DO_MPI call bml_shutdownF() diff --git a/src/prg_timer_mod.F90 b/src/prg_timer_mod.F90 index c480b0d0..15f5e3b7 100644 --- a/src/prg_timer_mod.F90 +++ b/src/prg_timer_mod.F90 @@ -203,7 +203,7 @@ end subroutine prg_timer_getid !> Done with timers subroutine prg_timer_shutdown() - deallocate(ptimer) + if(allocated(ptimer))deallocate(ptimer) end subroutine prg_timer_shutdown From 49154ca098dcd0ae2820fdee6340487cb11ca83e Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Thu, 12 Sep 2024 09:49:19 -0600 Subject: [PATCH 49/69] Prepare for OMP offload optimization --- examples/gpmdk/src/gpmdcov_dm_min.F90 | 1 - examples/gpmdk/src/gpmdcov_energandforces.F90 | 30 +++++++------- examples/gpmdk/src/gpmdcov_response.F90 | 40 +++++++++++-------- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index 44e37c76..1be67d20 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -283,7 +283,6 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) real(dp) :: tch1 real(8) :: mls_v, mls_coul, mls_mu, mls_red, mls_mix, mls_scfIter real(dp), allocatable :: KK0Res(:) - integer :: reclen converged = .false. if(.not.allocated(charges_old))allocate(charges_old(sy%nats)) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 855f06ea..0a2040d1 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -82,29 +82,27 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& integer, intent(in) :: norbs(:), sp(:), atnum real(dp), allocatable :: HPPP(:), HPPS(:), HSPS(:), HSSS(:), PPSMPP(:) real(dp), allocatable :: L(:), M(:), N(:) - real(dp), allocatable :: dr(:), dr_m(:), rab(:,:) + real(dp), allocatable, save :: dr(:), rab(:,:) + real(dp), allocatable :: dr_m(:) real(dp), allocatable :: onsites_m(:) real(dp), intent(inout) :: blk_out(:,:) - real(dp), allocatable :: blk(:,:) + real(dp), allocatable, save :: blk(:,:) real(dp), intent(in) :: refcoord(:),coord(:,:), lattice_vectors(:,:) real(dp), intent(in) :: onsites(:,:) real(dp), intent(in) :: intParams1(:,:,:),intParams2(:,:,:) - logical, allocatable :: dist_mask(:), onsite_mask(:), calc_mask(:), calcs_mask(:) - logical, allocatable :: calcsp_mask(:), param_mask(:,:), calc_mask_for_porbs(:) - logical, allocatable :: sorb_mask(:), pxorb_mask(:), pyorb_mask(:), pzorb_mask(:) - logical, allocatable :: sorb_at_mask(:), sporb_at_mask(:) - integer, allocatable :: atomidx(:), atomidx_m(:), orbidx(:), orbidx_m(:), orbidx_sel(:) - real(dp), allocatable :: intParams(:,:) + logical, allocatable,save :: dist_mask(:), onsite_mask(:), calc_mask(:), calcs_mask(:) + logical, allocatable,save :: calcsp_mask(:), param_mask(:,:), calc_mask_for_porbs(:) + logical, allocatable,save :: sorb_mask(:), pxorb_mask(:), pyorb_mask(:), pzorb_mask(:) + logical, allocatable,save :: sorb_at_mask(:), sporb_at_mask(:) + integer, allocatable,save :: atomidx(:), atomidx_m(:), orbidx(:), orbidx_m(:), orbidx_sel(:) + real(dp), allocatable,save :: intParams(:,:) nats = size(coord,dim=2) norbsall = sum(norbs) - allocate(blk(size(blk_out,dim=1),size(blk_out,dim=2))) - - blk(:,:)=0.0_dp - if(allocated(dr))then if(nats.ne.size(dr,dim=1))then + deallocate(blk) deallocate(dr) deallocate(atomidx) deallocate(orbidx) @@ -125,6 +123,7 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& endif if(.not.allocated(dr))then allocate(dr(nats)) + allocate(blk(4,size(blk_out,dim=2))) allocate(atomidx(nats)) atomidx = (/(i,i=1,nats)/) allocate(orbidx(norbsall)) @@ -176,7 +175,9 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& endif enddo endif - + + blk(:,:)=0.0_dp + do i = 1,3 Rab(:,i) = coord(i,:) Rab(:,i) = modulo((Rab(:,i) - refcoord(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) @@ -619,7 +620,6 @@ subroutine gpmdcov_EnergAndForces(charges) type(rankReduceData_t) :: mpimax_in(1), mpimax_out(1) integer :: k logical :: testsmd - call gpmdcov_msMem("gpmdcov","Before gpmd_EnergAndForces",lt%verbose,myRank) if(.not.allocated(coul_forces)) allocate(coul_forces(3,sy%nats)) @@ -781,7 +781,7 @@ subroutine gpmdcov_EnergAndForces(charges) GFSCOUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FSCOUL(:,i) SKForce(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%SKForce(:,i) enddo - + call bml_deallocate(dSx_bml) call bml_deallocate(dSy_bml) call bml_deallocate(dSz_bml) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 1e5f0408..82a22f8d 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -52,11 +52,14 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 ! real(PREC) :: P1(HDIM,HDIM) ! Density matrix response ! derivative with respect to perturbation H1 to H0 real(dp), intent(in) :: beta, mu0 ! Electronic temperature and chemicalpotential - real(dp), allocatable :: P1(:,:), DX1(:,:) + real(dp), allocatable :: P1(:,:) real(dp) :: h_0(HDIM), p_0(HDIM),p_02(HDIM),iD0(HDIM) real(dp) :: cnst, kB, mu1, sumdPdmu integer :: i, j, k character(20) :: bml_type + type(c_ptr) :: P1_bml_c_ptr + integer :: P1_bml_ld + real(c_double), pointer :: P1_bml_ptr(:,:) kB = 8.61739e-5 ! (eV/K) h_0 = evals ! Diagonal Hamiltonian H0 respresented in the eigenbasis Q @@ -66,30 +69,33 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 call nvtxStartRange("bml_copy_new",1) #endif call bml_copy_new(H1_bml,P1_bml) - call bml_copy_new(H1_bml,DX1_bml) #ifdef USE_NVTX call nvtxEndRange #endif call bml_scale(-cnst,P1_bml) !(set mu1 = 0 for simplicity) !Initialization of DM response in Q representation (not diagonal in Q) allocate(P1(HDIM,HDIM)) - allocate(DX1(HDIM,HDIM)) P1 = 0.0_dp - DX1 = 0.0_dp #ifdef USE_NVTX call nvtxStartRange("bml_export_to_dense",2) #endif - call bml_export_to_dense(P1_bml,P1) - call bml_export_to_dense(DX1_bml,DX1) #ifdef USE_NVTX call nvtxEndRange #endif !!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) #ifdef USE_OFFLOAD +! P1_bml_c_ptr = bml_get_data_ptr_dense(P1_bml) +! P1_bml_ld = bml_get_ld_dense(P1_bml) + + !!$omp target enter data map(alloc:p_0(1:HDIM),HDIM,j,k, P1_bml_ptr) !$omp target enter data map(alloc:P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) + !!$omp target update to(p_0(1:HDIM),HDIM) !$omp target update to(P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM) + !!$omp target data use_device_ptr(P1_bml_ptr) + !call c_f_pointer(P1_bml_c_ptr,P1_bml_ptr,shape=[HDIM,P1_bml_ld]) + #endif #ifdef USE_NVTX call nvtxStartRange("Response Kernel",3) @@ -98,17 +104,20 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !p_02 = p_0*p_0 #ifdef USE_OFFLOAD !$omp target teams distribute default(none) & - !!$acc parallel loop deviceptr(P1,DX1,p_0) + !!$acc parallel loop deviceptr(P1,p_0) !!$acc parallel loop - !$omp shared(HDIM,P1,p_0) + !!$omp shared(HDIM,P1_bml_ptr,p_0) + !$omp shared(HDIM,P1,p_0) + do k = 1,HDIM !$omp parallel do do j = 1,HDIM - P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) +! P1_bml_ptr(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1_bml_ptr(j,k) + 2.D0*(P1_bml_ptr(j,k)-(p_0(j) + p_0(k))*P1_bml_ptr(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & + &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) +! P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) enddo !$omp end parallel do - !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) - !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) enddo !!$acc end parallel loop !!$acc kernels deviceptr(p_0) @@ -142,16 +151,15 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !!$acc end data #ifdef USE_OFFLOAD !$omp target update from(P1(1:HDIM,1:HDIM)) - !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) + !!$omp end target data + !!$omp target exit data map(delete:p_0(1:HDIM),HDIM,j,k,P1_bml_ptr) + !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) #endif bml_type = bml_get_type(P1_bml) call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml - + deallocate(P1) - deallocate(DX1) - - call bml_deallocate(DX1_bml) dPdmu = beta*p_0*(1.D0-p_0) From 9fff9abdbb50363153bbd2ea92df365c17c1370e Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 13 Sep 2024 17:45:25 -0600 Subject: [PATCH 50/69] Fix bug in graph update --- examples/gpmdk/src/gpmdcov_part.F90 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index 4eddd4fa..d0e37f8a 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -35,11 +35,13 @@ subroutine gpmdcov_Part(ipreMD) endif if(ipreMD == 1)then + write(*,*)"DEBUG: Doing ipreMD graph partitioning at mdstep ",mdstep + call gpmdcov_msMem("gpmdcov_Part", "Before prg_get_covgraph",lt%verbose,myRank) call prg_get_covgraph(sy,nl%nnStruct,nl%nrnnstruct& ,gsp2%bml_type,gsp2%covgfact,g_bml,myMdim,lt%verbose) call gpmdcov_msMem("gpmdcov_Part", "After prg_get_covgraph",lt%verbose,myRank) - else + else !ipreMD == 1 #ifdef DO_MPI n_atoms = sy%nats max_updates = 100 @@ -90,11 +92,13 @@ subroutine gpmdcov_Part(ipreMD) #ifdef DO_MPI if (getNRanks() > 1) then call prg_barrierParallel - if((gsp2%parteach == 1) .or. (mod(mdstep,gsp2%parteach)==1) .or. (mdstep <= 1))then + if((gsp2%parteach == 1) .or. (mod(mdstep,gsp2%parteach)==0) .or. (mdstep <= 1))then call prg_sumIntReduceN(graph_p, myMdim*sy%nats) write(*,*)"DEBUG: Doing full graph reduction at mdstep ",mdstep graph_p_old = graph_p else + write(*,*)"DEBUG: Doing graph update reduction at mdstep ",mdstep + !MATLAB code ! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ! %% Detects new and removed edges between two graphs G1 and G2 %% @@ -261,7 +265,6 @@ subroutine gpmdcov_Part(ipreMD) write(*,*)"DEBUG: ktot = ",ktot !call prg_sumIntReduceN(graph_p, myMdim*sy%nats) - write(*,*)"DEBUG: Doing graph update reduction at mdstep ",mdstep ! %% Use G_removed and G_added to update from G1 to G2 call prg_sumIntReduceN(G_added,n_atoms*max_updates) call prg_sumIntReduceN(G_removed,n_atoms*max_updates) From 0315f8c07306f84af2c86cd5f927c6295e6228b2 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 17 Sep 2024 15:34:57 -0600 Subject: [PATCH 51/69] Use MAGMA pointer in offload response kernel --- examples/gpmdk/src/gpmdcov_response.F90 | 49 ++++++++----------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 82a22f8d..1dc7646a 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -73,6 +73,7 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 call nvtxEndRange #endif call bml_scale(-cnst,P1_bml) !(set mu1 = 0 for simplicity) !Initialization of DM response in Q representation (not diagonal in Q) +#ifndef USE_OFFLOAD allocate(P1(HDIM,HDIM)) P1 = 0.0_dp @@ -83,63 +84,49 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 #ifdef USE_NVTX call nvtxEndRange #endif +#endif ! USE_OFFLOAD -!!$acc data copy(P1(1:HDIM,1:HDIM),DX1(1:HDIM,1:HDIM),p_0(1:HDIM)) #ifdef USE_OFFLOAD -! P1_bml_c_ptr = bml_get_data_ptr_dense(P1_bml) -! P1_bml_ld = bml_get_ld_dense(P1_bml) + P1_bml_c_ptr = bml_get_data_ptr_dense(P1_bml) + P1_bml_ld = bml_get_ld_dense(P1_bml) - !!$omp target enter data map(alloc:p_0(1:HDIM),HDIM,j,k, P1_bml_ptr) - !$omp target enter data map(alloc:P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) - !!$omp target update to(p_0(1:HDIM),HDIM) - !$omp target update to(P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM) - !!$omp target data use_device_ptr(P1_bml_ptr) - !call c_f_pointer(P1_bml_c_ptr,P1_bml_ptr,shape=[HDIM,P1_bml_ld]) + !$omp target enter data map(alloc:p_0(1:HDIM)) + !$omp target update to(p_0(1:HDIM)) + + call c_f_pointer(P1_bml_c_ptr,P1_bml_ptr,shape=[P1_bml_ld,HDIM]) #endif #ifdef USE_NVTX call nvtxStartRange("Response Kernel",3) #endif do i = 1,m ! Loop over m recursion steps - !p_02 = p_0*p_0 #ifdef USE_OFFLOAD !$omp target teams distribute default(none) & - !!$acc parallel loop deviceptr(P1,p_0) - !!$acc parallel loop - !!$omp shared(HDIM,P1_bml_ptr,p_0) - !$omp shared(HDIM,P1,p_0) + !$omp shared(HDIM,P1_bml_ptr,p_0) do k = 1,HDIM !$omp parallel do do j = 1,HDIM -! P1_bml_ptr(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1_bml_ptr(j,k) + 2.D0*(P1_bml_ptr(j,k)-(p_0(j) + p_0(k))*P1_bml_ptr(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) - P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & - &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) -! P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) + 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + P1_bml_ptr(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1_bml_ptr(j,k) & + & + 2.D0*(P1_bml_ptr(j,k)-(p_0(j) + p_0(k))*P1_bml_ptr(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) enddo !$omp end parallel do enddo - !!$acc end parallel loop - !!$acc kernels deviceptr(p_0) - !!$acc kernels !$omp end target teams distribute !$omp target p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) !$omp end target #else - !$omp parallel do default(none) & - !$omp private(k,j) & + !$omp parallel do default(none) & + !$omp private(k,j) & !$omp shared(HDIM,P1,p_0) do k = 1,HDIM !$omp simd do j = 1,HDIM - !P1(:,k) = 1.D0/(2.D0*p_0(:)*(p_0(:)-1.D0)+1.D0)*((p_0(:) + p_0(k))*P1(:,k) + 2.D0*(P1(:,k)-(p_0(:) + p_0(k))*P1(:,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) enddo !$omp end simd - !P1(:,k) = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*(DX1(:,k) + 2.D0*(P1(:,k)-DX1(:,k))*1.D0/(2.D0*(p_0(k)*p_0(k)-p_0(k))+1.D0)*p_0(k)*p_0(k)) - !P1(k,:) = iD0(k)*(DX1(k,:) + 2.D0*(P1(k,:)-DX1(k,:))*p_0(:)) enddo !$omp end parallel do p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) @@ -148,18 +135,14 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 #ifdef USE_NVTX call nvtxEndRange #endif -!!$acc end data #ifdef USE_OFFLOAD - !$omp target update from(P1(1:HDIM,1:HDIM)) - !!$omp end target data - !!$omp target exit data map(delete:p_0(1:HDIM),HDIM,j,k,P1_bml_ptr) - !$omp target exit data map(delete:P1(1:HDIM,1:HDIM),p_0(1:HDIM),HDIM,j,k) -#endif + !$omp target exit data map(delete:p_0(1:HDIM)) +#else bml_type = bml_get_type(P1_bml) call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml - deallocate(P1) +#endif dPdmu = beta*p_0*(1.D0-p_0) From 9b5b4bd1fa6575d2c7929bba83cc067ff20845a6 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 29 Oct 2024 15:52:54 -0600 Subject: [PATCH 52/69] Working on openACC --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 32 +++++----- examples/gpmdk/src/gpmdcov_response.F90 | 60 ++++++++++++++----- 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 0a2040d1..f244e277 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -82,27 +82,29 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& integer, intent(in) :: norbs(:), sp(:), atnum real(dp), allocatable :: HPPP(:), HPPS(:), HSPS(:), HSSS(:), PPSMPP(:) real(dp), allocatable :: L(:), M(:), N(:) - real(dp), allocatable, save :: dr(:), rab(:,:) - real(dp), allocatable :: dr_m(:) + real(dp), allocatable :: dr(:), dr_m(:), rab(:,:) real(dp), allocatable :: onsites_m(:) real(dp), intent(inout) :: blk_out(:,:) - real(dp), allocatable, save :: blk(:,:) + real(dp), allocatable :: blk(:,:) real(dp), intent(in) :: refcoord(:),coord(:,:), lattice_vectors(:,:) real(dp), intent(in) :: onsites(:,:) real(dp), intent(in) :: intParams1(:,:,:),intParams2(:,:,:) - logical, allocatable,save :: dist_mask(:), onsite_mask(:), calc_mask(:), calcs_mask(:) - logical, allocatable,save :: calcsp_mask(:), param_mask(:,:), calc_mask_for_porbs(:) - logical, allocatable,save :: sorb_mask(:), pxorb_mask(:), pyorb_mask(:), pzorb_mask(:) - logical, allocatable,save :: sorb_at_mask(:), sporb_at_mask(:) - integer, allocatable,save :: atomidx(:), atomidx_m(:), orbidx(:), orbidx_m(:), orbidx_sel(:) - real(dp), allocatable,save :: intParams(:,:) + logical, allocatable :: dist_mask(:), onsite_mask(:), calc_mask(:), calcs_mask(:) + logical, allocatable :: calcsp_mask(:), param_mask(:,:), calc_mask_for_porbs(:) + logical, allocatable :: sorb_mask(:), pxorb_mask(:), pyorb_mask(:), pzorb_mask(:) + logical, allocatable :: sorb_at_mask(:), sporb_at_mask(:) + integer, allocatable :: atomidx(:), atomidx_m(:), orbidx(:), orbidx_m(:), orbidx_sel(:) + real(dp), allocatable :: intParams(:,:) nats = size(coord,dim=2) norbsall = sum(norbs) + allocate(blk(size(blk_out,dim=1),size(blk_out,dim=2))) + + blk(:,:)=0.0_dp + if(allocated(dr))then if(nats.ne.size(dr,dim=1))then - deallocate(blk) deallocate(dr) deallocate(atomidx) deallocate(orbidx) @@ -123,7 +125,6 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& endif if(.not.allocated(dr))then allocate(dr(nats)) - allocate(blk(4,size(blk_out,dim=2))) allocate(atomidx(nats)) atomidx = (/(i,i=1,nats)/) allocate(orbidx(norbsall)) @@ -175,9 +176,7 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& endif enddo endif - - blk(:,:)=0.0_dp - + do i = 1,3 Rab(:,i) = coord(i,:) Rab(:,i) = modulo((Rab(:,i) - refcoord(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) @@ -331,7 +330,7 @@ subroutine get_dH_or_dS_vect_local(dx,coords,hindex,spindex,intPairsH,onsitesH,s real(dp) :: Rax_m(3), Rax_p(3), Ray_m(3), Ray_p(3) real(dp) :: Raz_m(3), Raz_p(3), Rb(3), d, maxblockij real(dp), allocatable :: Rx(:), Ry(:), Rz(:), blockm(:,:,:) - real(dp), allocatable :: blockp(:,:,:), dh0(:,:), dH0x(:,:), dH0y(:,:), dH0z(:,:) + real(dp), allocatable :: blockp(:,:,:), dH0x(:,:), dH0y(:,:), dH0z(:,:) real(dp), intent(in) :: coords(:,:), dx, lattice_vectors(:,:), onsitesH(:,:) real(dp), intent(in) :: threshold type(bml_matrix_t), intent(inout) :: dH0x_bml, dH0y_bml, dH0z_bml @@ -620,6 +619,7 @@ subroutine gpmdcov_EnergAndForces(charges) type(rankReduceData_t) :: mpimax_in(1), mpimax_out(1) integer :: k logical :: testsmd + call gpmdcov_msMem("gpmdcov","Before gpmd_EnergAndForces",lt%verbose,myRank) if(.not.allocated(coul_forces)) allocate(coul_forces(3,sy%nats)) @@ -781,7 +781,7 @@ subroutine gpmdcov_EnergAndForces(charges) GFSCOUL(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%FSCOUL(:,i) SKForce(:,gpat%sgraph(ipt)%core_halo_index(i)+1) = syprt(ipt)%estr%SKForce(:,i) enddo - + call bml_deallocate(dSx_bml) call bml_deallocate(dSy_bml) call bml_deallocate(dSz_bml) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 1dc7646a..0059b726 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -86,37 +86,63 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 #endif #endif ! USE_OFFLOAD +#ifdef USE_NVTX + call nvtxStartRange("Response Kernel",3) +#endif + #ifdef USE_OFFLOAD P1_bml_c_ptr = bml_get_data_ptr_dense(P1_bml) P1_bml_ld = bml_get_ld_dense(P1_bml) - !$omp target enter data map(alloc:p_0(1:HDIM)) - !$omp target update to(p_0(1:HDIM)) - call c_f_pointer(P1_bml_c_ptr,P1_bml_ptr,shape=[P1_bml_ld,HDIM]) -#endif -#ifdef USE_NVTX - call nvtxStartRange("Response Kernel",3) -#endif + !call offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM) + +#ifdef USE_OMP + !$omp target enter data map(alloc:p_0(1:HDIM)) + !$omp target update to(p_0(1:HDIM)) +#else + !$acc enter data copyin(p_0(1:HDIM)) +#endif do i = 1,m ! Loop over m recursion steps -#ifdef USE_OFFLOAD +#ifdef USE_OMP !$omp target teams distribute default(none) & !$omp shared(HDIM,P1_bml_ptr,p_0) - +#else + !$acc parallel loop deviceptr(P1_bml_ptr) present(p_0) +#endif do k = 1,HDIM +#ifdef USE_OMP !$omp parallel do +#else + !$acc loop +#endif do j = 1,HDIM P1_bml_ptr(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1_bml_ptr(j,k) & & + 2.D0*(P1_bml_ptr(j,k)-(p_0(j) + p_0(k))*P1_bml_ptr(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) enddo +#ifdef USE_OMP !$omp end parallel do - enddo - !$omp end target teams distribute - !$omp target - p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) - !$omp end target #else + !$acc end loop +#endif + enddo +#ifdef USE_OMP + !$omp end target teams distribute + !$omp target +#else + !$acc end parallel + !$acc kernels present(p_0) +#endif + p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) +#ifdef USE_OMP + !$omp end target +#else + !$acc end kernels +#endif + enddo +#else + do i = 1,m ! Loop over m recursion steps !$omp parallel do default(none) & !$omp private(k,j) & !$omp shared(HDIM,P1,p_0) @@ -130,13 +156,17 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 enddo !$omp end parallel do p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) -#endif enddo +#endif #ifdef USE_NVTX call nvtxEndRange #endif #ifdef USE_OFFLOAD +#ifdef USE_OMP !$omp target exit data map(delete:p_0(1:HDIM)) +#else + !$acc exit data delete(p_0(1:HDIM)) +#endif #else bml_type = bml_get_type(P1_bml) From 829eb8773f4a7c1b21792056c6e9f099de4b9ab0 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 30 Oct 2024 21:25:37 -0600 Subject: [PATCH 53/69] Working offload of response using magma pointer --- examples/gpmdk/src/gpmdcov_response.F90 | 78 ++++++++++++++----------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 0059b726..4615715f 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -32,6 +32,7 @@ module gpmdcov_response_mod end type RespData_type public :: gpmdcov_response_dpdmu + private :: offload_kernel contains @@ -96,8 +97,44 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 call c_f_pointer(P1_bml_c_ptr,P1_bml_ptr,shape=[P1_bml_ld,HDIM]) - !call offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM) + call offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM,m) +#else + do i = 1,m ! Loop over m recursion steps + !$omp parallel do default(none) & + !$omp private(k,j) & + !$omp shared(HDIM,P1,p_0) + do k = 1,HDIM + !$omp simd + do j = 1,HDIM + P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & + &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) + enddo + !$omp end simd + enddo + !$omp end parallel do + p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) + enddo + bml_type = bml_get_type(P1_bml) + call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml + deallocate(P1) +#endif +#ifdef USE_NVTX + call nvtxEndRange +#endif + + dPdmu = beta*p_0*(1.D0-p_0) + + end subroutine gpmdcov_response_dpdmu + + subroutine offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM,m) + import :: C_DOUBLE + integer,value,intent(in) :: HDIM,P1_bml_ld,m + integer :: i,j,k + real(dp), intent(inout) :: p_0(HDIM) + real(C_DOUBLE),intent(inout) ::P1_bml_ptr(P1_bml_ld,HDIM) + + #ifdef USE_OMP !$omp target enter data map(alloc:p_0(1:HDIM)) !$omp target update to(p_0(1:HDIM)) @@ -131,7 +168,7 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !$omp end target teams distribute !$omp target #else - !$acc end parallel + !$acc end parallel loop !$acc kernels present(p_0) #endif p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) @@ -141,41 +178,12 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 !$acc end kernels #endif enddo -#else - do i = 1,m ! Loop over m recursion steps - !$omp parallel do default(none) & - !$omp private(k,j) & - !$omp shared(HDIM,P1,p_0) - do k = 1,HDIM - !$omp simd - do j = 1,HDIM - P1(j,k) = 1.D0/(2.D0*p_0(j)*(p_0(j)-1.D0)+1.D0)*((p_0(j) + p_0(k))*P1(j,k) & - &+ 2.D0*(P1(j,k)-(p_0(j) + p_0(k))*P1(j,k))*1.D0/(2.D0*p_0(k)*(p_0(k)-1.0D0)+1.D0)*p_0(k)*p_0(k)) - enddo - !$omp end simd - enddo - !$omp end parallel do - p_0 = 1.D0/(2.D0*(p_0(:)*p_0(:)-p_0(:))+1.D0)*p_0(:)*p_0(:) - enddo -#endif -#ifdef USE_NVTX - call nvtxEndRange -#endif -#ifdef USE_OFFLOAD #ifdef USE_OMP - !$omp target exit data map(delete:p_0(1:HDIM)) + !$omp target exit data map(from:p_0(1:HDIM)) #else - !$acc exit data delete(p_0(1:HDIM)) + !$acc exit data copyout(p_0(1:HDIM)) #endif -#else - - bml_type = bml_get_type(P1_bml) - call bml_import_from_dense(bml_type,P1,P1_bml,ZERO,HDIM) !Dense to dense_bml - deallocate(P1) -#endif - - dPdmu = beta*p_0*(1.D0-p_0) - - end subroutine gpmdcov_response_dpdmu + end subroutine offload_kernel + end module gpmdcov_response_mod From 578dec181990a17113fb521202380eac7189d77d Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Wed, 6 Nov 2024 09:59:28 -0700 Subject: [PATCH 54/69] fixing compilation bug gfortran --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 3 ++- examples/gpmdk/src/gpmdcov_response.F90 | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index f244e277..43b29377 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -179,7 +179,8 @@ subroutine get_SKBlock_vect_local(sp,refcoord,coord,lattice_vectors& do i = 1,3 Rab(:,i) = coord(i,:) - Rab(:,i) = modulo((Rab(:,i) - refcoord(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) + Rab(:,i) = modulo((Rab(:,i) - refcoord(i) + & + &0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) enddo dR(:) = norm2(Rab(:,:),dim=2) diff --git a/examples/gpmdk/src/gpmdcov_response.F90 b/examples/gpmdk/src/gpmdcov_response.F90 index 4615715f..4a54eb48 100644 --- a/examples/gpmdk/src/gpmdcov_response.F90 +++ b/examples/gpmdk/src/gpmdcov_response.F90 @@ -11,6 +11,7 @@ module gpmdcov_response_mod use prg_densitymatrix_mod use prg_openfiles_mod use gpmdcov_nvtx_mod + use ISO_C_BINDING, ONLY: C_DOUBLE implicit none @@ -128,7 +129,6 @@ subroutine gpmdcov_response_dpdmu(P1_bml,dPdMu,H1_bml,Norbs,beta,Q_bml,evals,mu0 end subroutine gpmdcov_response_dpdmu subroutine offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM,m) - import :: C_DOUBLE integer,value,intent(in) :: HDIM,P1_bml_ld,m integer :: i,j,k real(dp), intent(inout) :: p_0(HDIM) @@ -185,5 +185,5 @@ subroutine offload_kernel(p_0,P1_bml_ptr,P1_bml_ld,HDIM,m) #endif end subroutine offload_kernel - + end module gpmdcov_response_mod From bdc7776a90a8627d94673ffae98fc21f28475096 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 21 Nov 2024 15:47:44 -0700 Subject: [PATCH 55/69] Added voltage option --- examples/gpmdk/run/water/gpmd_input.in | 11 +- examples/gpmdk/run/water/voltage.vtg | 101 ++++++++++++++++++ examples/gpmdk/src/gpmdcov_initparts.F90 | 10 ++ examples/gpmdk/src/gpmdcov_nonequilibrium.F90 | 76 ++++++++++++- examples/gpmdk/src/gpmdcov_parser.F90 | 25 +++-- examples/gpmdk/src/gpmdcov_vars.F90 | 2 + 6 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 examples/gpmdk/run/water/voltage.vtg diff --git a/examples/gpmdk/run/water/gpmd_input.in b/examples/gpmdk/run/water/gpmd_input.in index 1ab0f32a..da1809fc 100644 --- a/examples/gpmdk/run/water/gpmd_input.in +++ b/examples/gpmdk/run/water/gpmd_input.in @@ -120,7 +120,7 @@ GSP2{ #PartitionCount= 256 #PartitionCount= 512 #PartitionCount= 16 - PartitionCount= 16 + PartitionCount= 1 #PartitionCount= 8 #PartitionCount= 1024 #PartitionCount= 32 @@ -129,7 +129,7 @@ GSP2{ #PartitionCount= 1 GraphThreshold= 0.05 ErrLimit= 1.0e-12 - PartEach= 3 + PartEach= 100 Mdim= -1 } @@ -150,7 +150,7 @@ XLBO{ JobName= XLBO Verbose= 1 Mprg_init= 2 - MaxSCFIter= 0 + MaxSCFIter= 100 MaxSCFInitIter= 50 NumThresh= 0.0 } @@ -173,9 +173,12 @@ GPMD{ DoVelocityRescale= F #VRFactor= 1.0 WriteTrajectory= T - WriteCoordsEach= 10 + WriteCoordsEach= 1 LangevinMethod= Siva LangevinDynamics= F LangevinGamma= 0.01 InitialTemperature= 300.0 + ApplyVoltage= T + VoltageFile= "voltage.vtg" } + diff --git a/examples/gpmdk/run/water/voltage.vtg b/examples/gpmdk/run/water/voltage.vtg new file mode 100644 index 00000000..33258f60 --- /dev/null +++ b/examples/gpmdk/run/water/voltage.vtg @@ -0,0 +1,101 @@ + 100 + 2 20.0 + 5 20.0 + 8 20.0 + 11 20.0 + 14 20.0 + 17 20.0 + 20 20.0 + 23 20.0 + 26 20.0 + 29 20.0 + 32 20.0 + 35 20.0 + 38 20.0 + 41 20.0 + 44 20.0 + 47 20.0 + 50 20.0 + 53 20.0 + 56 20.0 + 59 20.0 + 62 20.0 + 65 20.0 + 68 20.0 + 71 20.0 + 74 20.0 + 77 20.0 + 80 20.0 + 83 20.0 + 86 20.0 + 89 20.0 + 92 20.0 + 95 20.0 + 98 20.0 + 101 20.0 + 104 20.0 + 107 20.0 + 110 20.0 + 113 20.0 + 116 20.0 + 119 20.0 + 122 20.0 + 125 20.0 + 128 20.0 + 131 20.0 + 134 20.0 + 137 20.0 + 140 20.0 + 143 20.0 + 146 20.0 + 149 20.0 + 152 20.0 + 155 20.0 + 158 20.0 + 161 20.0 + 164 20.0 + 167 20.0 + 170 20.0 + 173 20.0 + 176 20.0 + 179 20.0 + 182 20.0 + 185 20.0 + 188 20.0 + 191 20.0 + 194 20.0 + 197 20.0 + 200 20.0 + 203 20.0 + 206 20.0 + 209 20.0 + 212 20.0 + 215 20.0 + 218 20.0 + 221 20.0 + 224 20.0 + 227 20.0 + 230 20.0 + 233 20.0 + 236 20.0 + 239 20.0 + 242 20.0 + 245 20.0 + 248 20.0 + 251 20.0 + 254 20.0 + 257 20.0 + 260 20.0 + 263 20.0 + 266 20.0 + 269 20.0 + 272 20.0 + 275 20.0 + 278 20.0 + 281 20.0 + 284 20.0 + 287 20.0 + 290 20.0 + 293 20.0 + 296 20.0 + 299 20.0 diff --git a/examples/gpmdk/src/gpmdcov_initparts.F90 b/examples/gpmdk/src/gpmdcov_initparts.F90 index e0bbfe95..869064e1 100644 --- a/examples/gpmdk/src/gpmdcov_initparts.F90 +++ b/examples/gpmdk/src/gpmdcov_initparts.F90 @@ -3,6 +3,7 @@ subroutine gpmdcov_InitParts use gpmdcov_vars use gpmdcov_writeout_mod + use gpmdcov_nonequilibrium_mod, only : gpmdcov_apply_voltage #ifdef USE_LATTE use gpmdcov_latte_mod #endif @@ -10,6 +11,8 @@ subroutine gpmdcov_InitParts use gpmdcov_nvtx_mod #endif + + implicit none integer :: norbsCore @@ -62,6 +65,8 @@ subroutine gpmdcov_InitParts call bml_deallocate(syprt(ipt)%estr%over) endif + + call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%ham0) call bml_noinit_matrix(lt%bml_type,bml_element_real,dp,norb,norb,syprt(ipt)%estr%over) #ifdef DO_MPI @@ -88,6 +93,11 @@ subroutine gpmdcov_InitParts endif #endif + if(gpmdt%applyv)then + call gpmdcov_apply_voltage(sy%nats,syprt(ipt)%nats,syprt(ipt)%estr%hindex,gpat%sgraph(ipt)%core_halo_index,& + &syprt(ipt)%estr%ham0,syprt(ipt)%estr%over) + endif + if (myRank == 1 .and. lt%verbose >= 5)then write(*,*)"H0 and S for part:" call bml_print_matrix("H0",syprt(ipt)%estr%ham0,0,6,0,6) diff --git a/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 b/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 index 896074a6..d1c9b3e1 100644 --- a/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 +++ b/examples/gpmdk/src/gpmdcov_nonequilibrium.F90 @@ -5,7 +5,7 @@ module gpmdcov_nonequilibrium_mod use gpmdcov_vars use prg_quantumdynamics_mod - public :: gpmdcov_get_currents + public :: gpmdcov_get_currents, gpmdcov_apply_voltage real(dp),parameter :: hbar = 0.65821188926_dp @@ -111,4 +111,78 @@ subroutine gpmdcov_get_currents(h_bml,p_bml,z_bml,subsyhindex,nats_core,& end subroutine gpmdcov_get_currents + + !> Applying a voltage (a shift in the chemical potential) + !! \brief This will apply a shift in the chemical potentials of preselected + !! atomic sites. + !! \param fullSystemNats Number of atoms for the full system + !! \param subSystemNats Number of atoms fro the subsystem + !! \param subsyhindex hindex for the subsystem + !! \param core_halo_index atomic indices for the core+halo subsystem + !! \param h_bml Non-scc hamiltonian for the subsystem + !! \param s_bml Overlap matrix for the subsystem + !! + subroutine gpmdcov_apply_voltage(fullSystemNats,subSystemNats,subsyhindex,core_halo_index,h_bml,s_bml) + + implicit none + + integer, allocatable, intent(in) :: subsyhindex(:,:) + character(2), allocatable :: symbols(:) + character(20) :: bml_type + integer :: ati, atj, ii, jj, nvpoints + integer :: nats_core, norbH, norbP, norbs, ich, chNats + integer, allocatable, intent(in) :: core_halo_index(:) + integer, intent(in) :: fullSystemNats, subSystemNats + real(dp) :: realtmp + real(dp), allocatable :: h_dense(:,:),s_dense(:,:),hamv(:),vspsv_dense(:,:) + type(bml_matrix_t) :: h_bml, s_bml + + if(.not. vinit)then + open(1,file=gpmdt%voltagef,status='OLD') + read(1,*)nvpoints + allocate(voltagev(fullSystemNats)) + do ii = 1,nvpoints + read(1,*)ati,realtmp + voltagev(ati) = realtmp + enddo + vinit = .true. + write(*,*)"Applying a shift in the chemical potentials using information in ",gpmdt%voltagef + endif + + norbs = bml_get_N(h_bml) + + allocate(h_dense(norbs,norbs)) + allocate(s_dense(norbs,norbs)) + allocate(vspsv_dense(norbs,norbs)) + allocate(hamv(norbs)) + hamv = 0.0_dp + call bml_export_to_dense(h_bml,h_dense) + call bml_export_to_dense(s_bml,s_dense) + + do ich = 1,subSystemNats + ati = core_halo_index(ich)+1 !Get the atom from the ch index + do ii = subsyhindex(1,ich),subsyhindex(2,ich) + hamv(ii) = voltagev(ati) + enddo + enddo + + do ii = 1,norbs + vspsv_dense(ii,:) = 0.5_dp*(hamv(ii)*s_dense(ii,:) + s_dense(i,:)*hamv(:)) + enddo + + deallocate(s_dense,hamv) + + h_dense = h_dense + vspsv_dense + + deallocate(vspsv_dense) + + bml_type = bml_get_type(h_bml) + + call bml_import_from_dense(bml_type, h_dense, h_bml, 0.0_dp, norbs) + + deallocate(h_dense) + + end subroutine gpmdcov_apply_voltage + + end module gpmdcov_nonequilibrium_mod diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 index cdba1426..e0f32b16 100644 --- a/examples/gpmdk/src/gpmdcov_parser.F90 +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -126,8 +126,13 @@ module gpmdcov_parser_mod !> Use Vectorized SKBlock method in PROGRESS logical :: usevectsk - - + + !> Apply a voltage on selceted atomic sites + logical :: applyv + + !> Voltage filename + character(100) :: voltagef + end type gpmd_type private @@ -151,16 +156,17 @@ subroutine gpmdcov_parse(filename,gpmdt) implicit none character(len=*), intent(in) :: filename type(gpmd_type), intent(inout) :: gpmdt - integer, parameter :: nkey_char = 4, nkey_int = 9, nkey_re = 7, nkey_log = 12 + integer, parameter :: nkey_char = 5, nkey_int = 9, nkey_re = 7, nkey_log = 13 integer :: i + real(dp) :: realtmp character(20) :: dummyc logical :: inGPMD !Library of keywords with the respective defaults. character(len=50), parameter :: keyvector_char(nkey_char) = [character(len=50) :: & - & 'JobName=', 'Var2C=', 'TrajectoryFormat=', 'LangevinMethod='] + & 'JobName=', 'Var2C=', 'TrajectoryFormat=', 'LangevinMethod=', 'VoltageFile='] character(len=100) :: valvector_char(nkey_char) = [character(len=100) :: & - &'MyMol', 'Def2', 'PDB', 'Goga'] + &'MyMol', 'Def2', 'PDB', 'Goga', 'None'] character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & & 'WriteCoordsEach=',"Var2I=","ReplicateX=","ReplicateY=","ReplicateZ=","PartsToTrack=",& @@ -177,10 +183,10 @@ subroutine gpmdcov_parse(filename,gpmdt) character(len=50), parameter :: keyvector_log(nkey_log) = [character(len=50) :: & &'DoVelocityRescale=','WriteResidueInTrajectory=','WriteTrajectory=','TrackReactivity=',& &'RestartFromDump=','UseLATTE=','HtoD=','LangevinDynamics=','UseSMD=', & - &'ComputeCurrents=', 'TranslateAndFoldToBox=', 'UseVectSKBlock='] + &'ComputeCurrents=', 'TranslateAndFoldToBox=', 'UseVectSKBlock=', 'ApplyVoltage='] logical :: valvector_log(nkey_log) = (/& &.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false., & - &.false.,.True.,.false./) + &.false.,.True.,.false.,.false./) !Start and stop characters character(len=50), parameter :: startstop(2) = [character(len=50) :: & @@ -293,6 +299,11 @@ subroutine gpmdcov_parse(filename,gpmdt) gpmdt%compcurr = valvector_log(10) gpmdt%trfl = valvector_log(11) gpmdt%usevectsk = valvector_log(12) + gpmdt%applyv = valvector_log(13) + + if(gpmdt%applyv)then + gpmdt%voltagef = valvector_char(5) + endif end subroutine gpmdcov_parse diff --git a/examples/gpmdk/src/gpmdcov_vars.F90 b/examples/gpmdk/src/gpmdcov_vars.F90 index 93695023..9155caaf 100644 --- a/examples/gpmdk/src/gpmdcov_vars.F90 +++ b/examples/gpmdk/src/gpmdcov_vars.F90 @@ -88,6 +88,7 @@ module gpmdcov_vars real(dp), allocatable :: n_5(:), onsitesH(:,:), onsitesS(:,:), rhoat(:) real(dp), allocatable :: origin(:), row(:), row1(:), auxcharge(:), auxcharge1(:) real(dp), allocatable :: g_dense(:,:),tch, Ker(:,:) + real(dp), allocatable :: voltagev(:) type(bml_matrix_t) :: aux_bml, dH0x_bml, dH0y_bml, dH0z_bml type(bml_matrix_t) :: dSx_bml, dSy_bml, dSz_bml, eigenvects type(bml_matrix_t) :: g_bml, ham0_bml, ham_bml @@ -119,6 +120,7 @@ module gpmdcov_vars logical, save :: lib_mode = .false. logical, save :: lib_init2 = .false. logical, save :: err_status = .false. + logical, save :: vinit = .false. type(bml_matrix_t) :: ZK1_bml, ZK2_bml, ZK3_bml type(bml_matrix_t) :: ZK4_bml, ZK5_bml, ZK6_bml From 7779af8e79db01e1015454bfb052cb65d234fc30 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Thu, 21 Nov 2024 18:25:00 -0700 Subject: [PATCH 56/69] Added dos --- examples/gpmdk/run/water/gpmd_input.in | 3 + examples/gpmdk/run/water/voltage.vtg | 402 ++++++++++++++++++------- examples/gpmdk/src/CMakeLists.txt | 1 + examples/gpmdk/src/gpmdcov_dos.F90 | 207 +++++++++++++ examples/gpmdk/src/gpmdcov_init.F90 | 5 +- examples/gpmdk/src/gpmdcov_mod.F90 | 19 ++ examples/gpmdk/src/gpmdcov_parser.F90 | 176 ++++++++++- examples/gpmdk/src/gpmdcov_part.F90 | 2 + examples/gpmdk/src/gpmdcov_vars.F90 | 1 + 9 files changed, 713 insertions(+), 103 deletions(-) create mode 100644 examples/gpmdk/src/gpmdcov_dos.F90 diff --git a/examples/gpmdk/run/water/gpmd_input.in b/examples/gpmdk/run/water/gpmd_input.in index da1809fc..1aa4ba96 100644 --- a/examples/gpmdk/run/water/gpmd_input.in +++ b/examples/gpmdk/run/water/gpmd_input.in @@ -182,3 +182,6 @@ GPMD{ VoltageFile= "voltage.vtg" } +ESTRUCTOUT{ +WriteTDOS= T +} diff --git a/examples/gpmdk/run/water/voltage.vtg b/examples/gpmdk/run/water/voltage.vtg index 33258f60..d346102f 100644 --- a/examples/gpmdk/run/water/voltage.vtg +++ b/examples/gpmdk/run/water/voltage.vtg @@ -1,101 +1,301 @@ - 100 - 2 20.0 - 5 20.0 - 8 20.0 - 11 20.0 - 14 20.0 - 17 20.0 - 20 20.0 - 23 20.0 - 26 20.0 - 29 20.0 - 32 20.0 - 35 20.0 - 38 20.0 - 41 20.0 - 44 20.0 - 47 20.0 - 50 20.0 - 53 20.0 - 56 20.0 - 59 20.0 - 62 20.0 - 65 20.0 - 68 20.0 - 71 20.0 - 74 20.0 - 77 20.0 - 80 20.0 - 83 20.0 - 86 20.0 - 89 20.0 - 92 20.0 - 95 20.0 - 98 20.0 - 101 20.0 - 104 20.0 - 107 20.0 - 110 20.0 - 113 20.0 - 116 20.0 - 119 20.0 - 122 20.0 - 125 20.0 - 128 20.0 - 131 20.0 - 134 20.0 - 137 20.0 - 140 20.0 - 143 20.0 - 146 20.0 - 149 20.0 - 152 20.0 - 155 20.0 - 158 20.0 - 161 20.0 - 164 20.0 - 167 20.0 - 170 20.0 - 173 20.0 - 176 20.0 - 179 20.0 - 182 20.0 - 185 20.0 - 188 20.0 - 191 20.0 - 194 20.0 - 197 20.0 - 200 20.0 - 203 20.0 - 206 20.0 - 209 20.0 - 212 20.0 - 215 20.0 - 218 20.0 - 221 20.0 - 224 20.0 - 227 20.0 - 230 20.0 - 233 20.0 - 236 20.0 - 239 20.0 - 242 20.0 - 245 20.0 - 248 20.0 - 251 20.0 - 254 20.0 - 257 20.0 - 260 20.0 - 263 20.0 - 266 20.0 - 269 20.0 - 272 20.0 - 275 20.0 - 278 20.0 - 281 20.0 - 284 20.0 - 287 20.0 - 290 20.0 - 293 20.0 - 296 20.0 - 299 20.0 + 300 + 1 10.0 + 2 10.0 + 3 10.0 + 4 10.0 + 5 10.0 + 6 10.0 + 7 10.0 + 8 10.0 + 9 10.0 + 10 10.0 + 11 10.0 + 12 10.0 + 13 10.0 + 14 10.0 + 15 10.0 + 16 10.0 + 17 10.0 + 18 10.0 + 19 10.0 + 20 10.0 + 21 10.0 + 22 10.0 + 23 10.0 + 24 10.0 + 25 10.0 + 26 10.0 + 27 10.0 + 28 10.0 + 29 10.0 + 30 10.0 + 31 10.0 + 32 10.0 + 33 10.0 + 34 10.0 + 35 10.0 + 36 10.0 + 37 10.0 + 38 10.0 + 39 10.0 + 40 10.0 + 41 10.0 + 42 10.0 + 43 10.0 + 44 10.0 + 45 10.0 + 46 10.0 + 47 10.0 + 48 10.0 + 49 10.0 + 50 10.0 + 51 10.0 + 52 10.0 + 53 10.0 + 54 10.0 + 55 10.0 + 56 10.0 + 57 10.0 + 58 10.0 + 59 10.0 + 60 10.0 + 61 10.0 + 62 10.0 + 63 10.0 + 64 10.0 + 65 10.0 + 66 10.0 + 67 10.0 + 68 10.0 + 69 10.0 + 70 10.0 + 71 10.0 + 72 10.0 + 73 10.0 + 74 10.0 + 75 10.0 + 76 10.0 + 77 10.0 + 78 10.0 + 79 10.0 + 80 10.0 + 81 10.0 + 82 10.0 + 83 10.0 + 84 10.0 + 85 10.0 + 86 10.0 + 87 10.0 + 88 10.0 + 89 10.0 + 90 10.0 + 91 10.0 + 92 10.0 + 93 10.0 + 94 10.0 + 95 10.0 + 96 10.0 + 97 10.0 + 98 10.0 + 99 10.0 + 100 10.0 + 101 10.0 + 102 10.0 + 103 10.0 + 104 10.0 + 105 10.0 + 106 10.0 + 107 10.0 + 108 10.0 + 109 10.0 + 110 10.0 + 111 10.0 + 112 10.0 + 113 10.0 + 114 10.0 + 115 10.0 + 116 10.0 + 117 10.0 + 118 10.0 + 119 10.0 + 120 10.0 + 121 10.0 + 122 10.0 + 123 10.0 + 124 10.0 + 125 10.0 + 126 10.0 + 127 10.0 + 128 10.0 + 129 10.0 + 130 10.0 + 131 10.0 + 132 10.0 + 133 10.0 + 134 10.0 + 135 10.0 + 136 10.0 + 137 10.0 + 138 10.0 + 139 10.0 + 140 10.0 + 141 10.0 + 142 10.0 + 143 10.0 + 144 10.0 + 145 10.0 + 146 10.0 + 147 10.0 + 148 10.0 + 149 10.0 + 150 10.0 + 151 10.0 + 152 10.0 + 153 10.0 + 154 10.0 + 155 10.0 + 156 10.0 + 157 10.0 + 158 10.0 + 159 10.0 + 160 10.0 + 161 10.0 + 162 10.0 + 163 10.0 + 164 10.0 + 165 10.0 + 166 10.0 + 167 10.0 + 168 10.0 + 169 10.0 + 170 10.0 + 171 10.0 + 172 10.0 + 173 10.0 + 174 10.0 + 175 10.0 + 176 10.0 + 177 10.0 + 178 10.0 + 179 10.0 + 180 10.0 + 181 10.0 + 182 10.0 + 183 10.0 + 184 10.0 + 185 10.0 + 186 10.0 + 187 10.0 + 188 10.0 + 189 10.0 + 190 10.0 + 191 10.0 + 192 10.0 + 193 10.0 + 194 10.0 + 195 10.0 + 196 10.0 + 197 10.0 + 198 10.0 + 199 10.0 + 200 10.0 + 201 10.0 + 202 10.0 + 203 10.0 + 204 10.0 + 205 10.0 + 206 10.0 + 207 10.0 + 208 10.0 + 209 10.0 + 210 10.0 + 211 10.0 + 212 10.0 + 213 10.0 + 214 10.0 + 215 10.0 + 216 10.0 + 217 10.0 + 218 10.0 + 219 10.0 + 220 10.0 + 221 10.0 + 222 10.0 + 223 10.0 + 224 10.0 + 225 10.0 + 226 10.0 + 227 10.0 + 228 10.0 + 229 10.0 + 230 10.0 + 231 10.0 + 232 10.0 + 233 10.0 + 234 10.0 + 235 10.0 + 236 10.0 + 237 10.0 + 238 10.0 + 239 10.0 + 240 10.0 + 241 10.0 + 242 10.0 + 243 10.0 + 244 10.0 + 245 10.0 + 246 10.0 + 247 10.0 + 248 10.0 + 249 10.0 + 250 10.0 + 251 10.0 + 252 10.0 + 253 10.0 + 254 10.0 + 255 10.0 + 256 10.0 + 257 10.0 + 258 10.0 + 259 10.0 + 260 10.0 + 261 10.0 + 262 10.0 + 263 10.0 + 264 10.0 + 265 10.0 + 266 10.0 + 267 10.0 + 268 10.0 + 269 10.0 + 270 10.0 + 271 10.0 + 272 10.0 + 273 10.0 + 274 10.0 + 275 10.0 + 276 10.0 + 277 10.0 + 278 10.0 + 279 10.0 + 280 10.0 + 281 10.0 + 282 10.0 + 283 10.0 + 284 10.0 + 285 10.0 + 286 10.0 + 287 10.0 + 288 10.0 + 289 10.0 + 290 10.0 + 291 10.0 + 292 10.0 + 293 10.0 + 294 10.0 + 295 10.0 + 296 10.0 + 297 10.0 + 298 10.0 + 299 10.0 + 300 10.0 diff --git a/examples/gpmdk/src/CMakeLists.txt b/examples/gpmdk/src/CMakeLists.txt index 7356e59c..c0f2a333 100644 --- a/examples/gpmdk/src/CMakeLists.txt +++ b/examples/gpmdk/src/CMakeLists.txt @@ -166,6 +166,7 @@ list( "gpmdcov_field.F90" "gpmdcov_nvtx.F90" "gpmdcov_response.F90" +"gpmdcov_dos.F90" ) diff --git a/examples/gpmdk/src/gpmdcov_dos.F90 b/examples/gpmdk/src/gpmdcov_dos.F90 new file mode 100644 index 00000000..3cd429b4 --- /dev/null +++ b/examples/gpmdk/src/gpmdcov_dos.F90 @@ -0,0 +1,207 @@ +!> Computes density of state (DOS) for molecules or 2yy +!! user-defined groups of atoms within a molecular system +module gpmdcov_dos_mod +use bml +contains + !> Computes density of state (DOS) for a moleculer system + !! + subroutine compute_dos(num_points, sigma, emin, emax, evals, mu, filename) + use prg_openfiles_mod + + implicit none + integer, parameter :: dp = kind(1.0d0) + integer, intent(in) :: num_points + real(dp), intent(in) :: sigma, emin, emax, mu + real(dp), allocatable, intent(in) :: evals(:) + character(*), intent(in) :: filename + + integer :: fileunit, i, j, nevals + real(dp), allocatable :: dos(:) + real(dp) :: ecut, de, eps + + !write(*,*) "gpmdcov_dos_mod: Input variables ",num_points,sigma,& + ! &emin,emax,filename + !> + !! Open output file + call prg_open_file(fileunit, filename) + + ecut = 2.0_dp + + allocate(dos(num_points)) + nevals = size(evals) + de = (emax - emin) / real(num_points) + !> + !! Compute all contributions from all eigenvalues (evals) + !! For every point, compute the Gaussian contributions + !! of each eval + dos = 0.0_dp + do i = 1, num_points + eps = emin + de * (i - 1) + do j = 1, nevals + dos(i) = dos(i) + gaussian(eps, evals(j), sigma) + enddo + write(fileunit,*) eps - mu, dos(i) + enddo + + deallocate(dos) + close(fileunit) + + end subroutine compute_dos + + !> Computes local density of state (DOS) for a user-defined group of atoms + !! within a moleculer system + !! + subroutine compute_local_dos(num_points, atoms, hindex, sigma, emin, & + &emax, evects, evals, overlaps, mu, zmat_bml, symbols, lfilename) + use prg_openfiles_mod + + implicit none + integer, parameter :: dp = kind(1.0d0) + integer, intent(in) :: num_points + integer, allocatable :: atoms(:) + integer, allocatable, intent(in) :: hindex(:,:) + real(dp), intent(in) :: sigma, emin, emax, mu + real(dp), allocatable, intent(in) :: evals(:) + Type(bml_matrix_t), intent(in) :: evects, overlaps, zmat_bml + character(*), intent(in) :: symbols(:) + character(*), intent(in) :: lfilename + + integer :: fileunit, a, i, j, k, m, nevals, norb, nstates, at + real(dp), allocatable :: dos(:), row(:), weights_dense(:,:), weights_2(:,:) + real(dp), allocatable :: subsystem_weight(:), atom_weights(:) + real(dp), allocatable :: evects_dense(:,:), overlaps_dense(:,:) + !real(dp) :: ecut, de, eps, sum_squares, subsystem_weight + real(dp) :: ecut, de, eps, sum_squares + Type(bml_matrix_t) :: weights_matrix,z_transpose, zc_matrix + + !! Allocate row matrix and dense versions of eivenvectors(C), overlaps (S), and weigths matricies + allocate(row(size(evals))) + allocate(subsystem_weight(size(evals))) + nstates = bml_get_N(evects) + allocate(weights_dense(nstates,nstates)) + allocate(weights_2(nstates,nstates)) + allocate(evects_dense(nstates,nstates)) + allocate(overlaps_dense(nstates,nstates)) + + !! Create new bml matricies for SC product and weights based on the number of orbitals + call bml_zero_matrix("dense",bml_element_real,dp,nstates,nstates,weights_matrix) + call bml_zero_matrix("dense",bml_element_real,dp,nstates,nstates,zc_matrix) + + call bml_print_matrix("evects",evects,0,6,0,6) + call bml_print_matrix("overlap",overlaps,0,6,0,6) + !call bml_get_row(evects,1,row) + !write(*,*) "Row of evects ",row + + !! Transpose Z matrix (Z) + call bml_transpose_new(zmat_bml, z_transpose) + !! Multiply Z matrix (Z) and eigenvectors (C) to form zc_matrix + call bml_multiply(z_transpose,evects,zc_matrix,1.0_dp,0.0_dp,0.0_dp) + + !! Multiply overlaps matrix (S) and ZC matrix to form weights_matrix + call bml_multiply(zc_matrix,overlaps,weights_matrix,1.0_dp,0.0_dp,0.0_dp) + + !! Print new weights matrix to check + !call bml_print_matrix("weights",weights_matrix,0,6,0,6) + + !! Create dense matrices from BML matricies + call bml_export_to_dense(overlaps,overlaps_dense) + call bml_export_to_dense(evects,evects_dense) + call bml_export_to_dense(weights_matrix,weights_dense) + weights_2 = matmul(evects_dense, evects_dense) + + sum_squares = 0.0_dp + do i = 1,nstates + sum_squares = sum_squares + weights_dense(2,i)**2 + !sum_squares = sum_squares + evects_dense(2,i)**2 + enddo + write(*,*) "Sum Squares ",sum_squares + + !! Open output file + call prg_open_file(fileunit, lfilename) + + !! Compute values of w_i + subsystem_weight=0.0_dp + + !do i = 1,size(atoms) + do k = 1,nstates + do a = 1,size(atoms) + !if (symbols(atoms(i)) .eq. "C") then + do j = hindex(1,atoms(a)),hindex(2,atoms(a)) + subsystem_weight(k) = subsystem_weight(k) + weights_dense(k,j)**2 + !subsystem_weight(k) = subsystem_weight(k) + evects_dense(k,j)**2 + write(*,*) "Add to subsystem weight ",weights_dense(k,j)**2 + !write(*,*) "Add to subsystem weight ",evects_dense(k,j)**2 + enddo + !endif + enddo + enddo + + write(*,*) "Subsystem weight ",subsystem_weight + + ecut = 2.0_dp + + allocate(dos(num_points)) + nevals = size(evals) + write(*,*) "Nstates and nevals ",nstates,nevals + de = (emax - emin) / real(num_points,dp) + !> + !! Compute all contributions from all eigenvalues (evals) + !! For every point, compute the Gaussian contributions + !! of each eval + dos = 0.0_dp + do m = 1, num_points + eps = emin + de * (m - 1) + do j = 1, nevals + ! dos(i) = dos(i) + gaussian(eps, evals(j), sigma) + dos(m) = dos(m) + gaussian(eps, evals(j), sigma) * subsystem_weight(j) + + enddo + write(fileunit,*) eps - mu, dos(m) + enddo + + !> + ! Print out weights of different atoms + ! Want to see what all atoms have the most weight right around Fermi level + !allocate(atom_weights(sy%nats)) + !atom_weights = 0.0_dp + + !do k = 1, nstates + ! do at = 1,sy%nats + ! do i = hindex(at,1), hindex(at,2) + ! atom_weights(at) = atom_weights(at) + weights_dense(k,j) + ! enddo + ! enddo + !enddo + + !do at = 1,sy%nats + ! write(*,*) "Atom weight ",at,atom_weights(at) + !enddo + + !deallocate(atom_weights) + deallocate(dos) + close(fileunit) + + end subroutine compute_local_dos + + !> Function to compute the Gaussian contibution for an eigenvalue + !! + function gaussian(eps, eps_i, sigma) result(G) + implicit none + integer, parameter :: dp = kind(1.0d0) + real(dp), intent(in) :: eps, eps_i, sigma + real(dp) :: pi, delta, arg, myexp + real(dp) :: G + delta = 10E-5 + pi = 3.1415926535897932384626_dp + arg = ((eps - eps_i)/sigma)**2 + if (arg .lt. 1E-5) then + myexp = 1.0_dp + write(*,*) "myexp 1.0", arg + else + myexp = exp(-0.5*arg) + write(*,*) "myexp ",myexp,arg + endif + G = 1.0 / (sigma * sqrt(2.0 * pi)) * myexp + end function gaussian + +end module gpmdcov_dos_mod diff --git a/examples/gpmdk/src/gpmdcov_init.F90 b/examples/gpmdk/src/gpmdcov_init.F90 index b52986cb..7e77e5cf 100644 --- a/examples/gpmdk/src/gpmdcov_init.F90 +++ b/examples/gpmdk/src/gpmdcov_init.F90 @@ -80,7 +80,10 @@ subroutine gpmdcov_Init(lib_on) !> Parsing specific variales for the gpmd code call gpmdcov_parse(trim(adjustl(inputfile)),gpmdt) - + + !> Parsing specific variales for controlling electronic structure output + call gpmdcov_estructout_parse(trim(adjustl(inputfile)),estrout) + !> Parse variables for the kernel method call gpmdcov_parseKernel(kernel,inputfile) endif diff --git a/examples/gpmdk/src/gpmdcov_mod.F90 b/examples/gpmdk/src/gpmdcov_mod.F90 index b6231cb4..d6f8b71b 100644 --- a/examples/gpmdk/src/gpmdcov_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_mod.F90 @@ -77,6 +77,7 @@ subroutine gpmdcov_muFromParts() use gpmdcov_vars use gpmdcov_writeout_mod + use gpmdcov_dos_mod integer :: mycount real(dp) :: mls_mu @@ -191,6 +192,24 @@ subroutine gpmdcov_muFromParts() call gpmdcov_msI("gpmdcov_muFromParts","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) call gpmdcov_msI("gpmdcov_muFromParts","Time for mu search="//to_string(mls() - mls_mu),lt%verbose,myRank) + if (estrout%write_tdos .and. MDstep .eq. 1) then + if (gpat%TotalParts .gt. 1) then + call gpmdcov_msI("gpmdcov_muFromParts","For computing DOS, TotalParts should be 1 ",lt%verbose,myRank) + stop + endif + call compute_dos(estrout%tdos_num_points, estrout%tdos_sigma, estrout%tdos_emin,& + &estrout%tdos_emax, evalsAll, Ef, estrout%tdos_output_filename) + if (estrout%compute_pdos) then + call gpmdcov_msI("gpmdcov_muFromParts","Computing PDOS ",lt%verbose,myRank) + !write(*,*) "Hindex ",syprt(1)%estr%hindex + call compute_local_dos(estrout%tdos_num_points, estrout%pdos_atoms, syprt(1)%estr%hindex, estrout%tdos_sigma,& + &estrout%tdos_emin, estrout%tdos_emax,syprt(1)%estr%evects,evalsAll,syprt(1)%estr%over, Ef,& + &syprt(1)%estr%zmat, syprt(1)%symbol, estrout%pdos_output_filename) + endif + write(*,*) "gpmdcov_mod: called compute_dos to compute TDOS" + stop + endif + end subroutine gpmdcov_muFromParts !> Perform musearch. diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 index e0f32b16..56c5e582 100644 --- a/examples/gpmdk/src/gpmdcov_parser.F90 +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -135,9 +135,48 @@ module gpmdcov_parser_mod end type gpmd_type + !> electrontic structure output type + !! Controls output of electronic structure information + !! as collected across QMD simulation + type, public :: estructout_type + !> TDOS Output file name + character (len=100) :: tdos_output_filename + + !> PDOS Output file name + character (len=100) :: pdos_output_filename + + !> Write out Density of State + logical :: write_tdos + + !> Compute Projected Density of State + logical :: compute_pdos + + !> Write out Projected Density of State + logical :: write_pdos + + !> Left boundary of DOS energy window + real(dp) :: tdos_emin + + !> Right boundary of DOS energy window + real(dp) :: tdos_emax + + !> TDOS sigma value + real(dp) :: tdos_sigma + + !> TDOS Number of Points + integer :: tdos_num_points + + !> PDOS Number of Atoms in Subsystem + integer :: pdos_num_atoms + + !> PDOS Atom Numbers + integer, allocatable :: pdos_atoms(:) + + end type estructout_type + private - public :: gpmdcov_parse + public :: gpmdcov_parse, gpmdcov_estructout_parse contains @@ -307,4 +346,139 @@ subroutine gpmdcov_parse(filename,gpmdt) end subroutine gpmdcov_parse + + !> Electronic sturcture output control parser. + !! \brief This module is used to parse all the input variables for this program. + !! Adding a new input keyword to the parser: + !! - If the variable is real, we have to increase nkey_re. + !! - Add the keyword (character type) in the keyvector_re vector. + !! - Add a default value (real type) in the valvector_re. + !! - Define a new variable and pass the value through valvector_re(num) + !! where num is the position of the new keyword in the vector. + !! \param filename File name for the input. + !! \param gpmd type. + !! + subroutine gpmdcov_estructout_parse(filename,estrout) + implicit none + character(len=*), intent(in) :: filename + type(estructout_type), intent(inout) :: estrout + integer, parameter :: nkey_char = 2, nkey_int = 2, nkey_re = 3, nkey_log = 3 + integer :: i, counter, ind, j, startatom, endatom, single + character(20) :: dummyc + character(20) :: oneline + logical :: inESTR + + !Library of keywords with the respective defaults. + character(len=50), parameter :: keyvector_char(nkey_char) = [character(len=50) :: & + & 'TDOSOutputFileName=', 'PDOSOutputFileName='] + character(len=100) :: valvector_char(nkey_char) = [character(len=100) :: & + &'tdos_output', 'pdos_output'] + + character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & + & 'TDOSNumPoints=', 'PDOSNumAtoms='] + integer :: valvector_int(nkey_int) = (/ & + & 1000, 0/) + + character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & + & 'TDOSEmin=', 'TDOSEmax=', 'TDOSSigma='] + real(dp) :: valvector_re(nkey_re) = (/& + & -20.0_dp, 20.0_dp, 0.5_dp/) + + character(len=50), parameter :: keyvector_log(nkey_log) = [character(len=50) :: & + &'WriteTDOS=', 'ComputePDOS=', 'WritePDOS='] + logical :: valvector_log(nkey_log) = (/& + &.false., .false., .false./) + + !Start and stop characters + character(len=50), parameter :: startstop(2) = [character(len=50) :: & + 'ESTRUCTOUT{', '}'] + + call prg_parsing_kernel(keyvector_char,valvector_char& + ,keyvector_int,valvector_int,keyvector_re,valvector_re,& + keyvector_log,valvector_log,trim(filename),startstop) + + !Characters + estrout%tdos_output_filename = valvector_char(1) + estrout%pdos_output_filename = valvector_char(2) + + !Integer + estrout%tdos_num_points = valvector_int(1) + estrout%pdos_num_atoms = valvector_int(2) + inESTR = .false. + if(estrout%pdos_num_atoms > 0) then + write(*,*) "Reading PDOS atoms" + open(1, file=trim(filename)) + do i = 1,10000 + read(1,*) dummyc + !write(*,*) trim(adjustl(dummyc)) + if(trim(adjustl(dummyc)) == "PDOSAtoms[")then + exit + end if + if(trim(adjustl(dummyc)) == "ESTRUCTOUT{") inESTR = .true. + + if(trim(dummyc) == "}" .and. inESTR)then + write(*,*)'ERROR: No PDOS Atoms defined' + write(*,*)"Here is an example block you should add to the" + write(*,*)"input file" + write(*,*)"" + write(*,*)"PDOSAtoms[" + write(*,*)" 1:10" + write(*,*)" 12" + write(*,*)" 15:30" + write(*,*)"]" + write(*,*)"" + write(*,*)"This will assign atoms 1 through 10, 12, and 15" + write(*,*)"through 30 to the PDOS subsystem" + stop + end if + end do + allocate(estrout%pdos_atoms(estrout%pdos_num_atoms)) + counter=1 + do i = 1,estrout%pdos_num_atoms + !! Read through PDOS atom lines: maximum number of lines is number of atoms + !! if each atom number is on its own line + read(1,*) oneline + !! Exit if the end of the PDOSAtoms bracket is reached + !! This will happen if one or more atom groups is given + if(trim(adjustl(oneline)) == "]") then + exit + endif + !! If an atom group is given, there will be a colon separating the + !! first and last atom number in the group + ind = index(oneline,":") + if(ind .gt. 0) then + read(oneline(1:ind-1), '(i5)') startatom + read(oneline(ind+1:len(oneline)), '(i5)') endatom + do j = startatom, endatom + !write(*,*) "Adding atom to array ",j + estrout%pdos_atoms(counter) = j + counter = counter + 1 + enddo + else + !write(*,*) "Adding single atom to array ",oneline + read(oneline,'(i5)') single + estrout%pdos_atoms(counter) = single + counter = counter + 1 + endif + end do + write(*,*)"" + close(1) + endif + write(*,*) "PDOS Atoms ",estrout%pdos_atoms + !call gpmdcov_msI("gpmdcov_parser","PDOS Atom Array " & + ! & // to_string(estrout%pdos_atoms),lt%verbose,myRank) + + + !Reals + estrout%tdos_emin = valvector_re(1) + estrout%tdos_emax = valvector_re(2) + estrout%tdos_sigma = valvector_re(3) + + !Logs + estrout%write_tdos = valvector_log(1) + estrout%compute_pdos = valvector_log(2) + estrout%write_pdos = valvector_log(3) + + end subroutine gpmdcov_estructout_parse + end module gpmdcov_parser_mod diff --git a/examples/gpmdk/src/gpmdcov_part.F90 b/examples/gpmdk/src/gpmdcov_part.F90 index d0e37f8a..efc99d23 100644 --- a/examples/gpmdk/src/gpmdcov_part.F90 +++ b/examples/gpmdk/src/gpmdcov_part.F90 @@ -40,6 +40,8 @@ subroutine gpmdcov_Part(ipreMD) call gpmdcov_msMem("gpmdcov_Part", "Before prg_get_covgraph",lt%verbose,myRank) call prg_get_covgraph(sy,nl%nnStruct,nl%nrnnstruct& ,gsp2%bml_type,gsp2%covgfact,g_bml,myMdim,lt%verbose) + + call gpmdcov_msMem("gpmdcov_Part", "After prg_get_covgraph",lt%verbose,myRank) else !ipreMD == 1 #ifdef DO_MPI diff --git a/examples/gpmdk/src/gpmdcov_vars.F90 b/examples/gpmdk/src/gpmdcov_vars.F90 index 9155caaf..d0d350c9 100644 --- a/examples/gpmdk/src/gpmdcov_vars.F90 +++ b/examples/gpmdk/src/gpmdcov_vars.F90 @@ -109,6 +109,7 @@ module gpmdcov_vars type(tbparams_type) :: tb type(xlbo_type) :: xl type(gpmd_type) :: gpmdt + type(estructout_type) :: estrout logical :: first_part = .true. logical :: converged = .false. logical :: firstKernel = .true. From c5dc023a765221510da42a53e4e7b11036a63462 Mon Sep 17 00:00:00 2001 From: Christian Francisco Negre Date: Wed, 4 Dec 2024 11:57:56 -0700 Subject: [PATCH 57/69] changes for coarse MD --- examples/gpmdk/src/gpmdcov_dm_min.F90 | 19 +++++++++---- examples/gpmdk/src/gpmdcov_mdloop.F90 | 23 +++++++++++++-- examples/gpmdk/src/gpmdcov_parser.F90 | 36 ++++++++++++++++++------ examples/gpmdk/src/gpmdcov_reshuffle.F90 | 5 ++++ 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_dm_min.F90 b/examples/gpmdk/src/gpmdcov_dm_min.F90 index 1be67d20..31a7fd7b 100644 --- a/examples/gpmdk/src/gpmdcov_dm_min.F90 +++ b/examples/gpmdk/src/gpmdcov_dm_min.F90 @@ -179,6 +179,7 @@ subroutine gpmdcov_DM_Min(Nr_SCF,nguess,mix) else call prg_qmixer(nguess,charges_old,dqin,& dqout,scferror,iscf,lt%pulaycoeff,lt%mpulay,0) + scferror = scferror/sqrt(real(sy%nats,dp)) call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Pulay/DIIS mixing ",lt%verbose,myRank) endif else @@ -207,14 +208,19 @@ subroutine gpmdcov_DM_Min(Nr_SCF,nguess,mix) ! endif if(converged)then ! To do a last extra step. + if (myRank == 1) then + write(*,*)""; write(*,*)"SCF converged within",iscf,"steps ..." + write(*,*)"SCF error =",scferror + endif exit else - if(scferror < lt%scftol .and. iscf > 1) then + if(scferror < lt%scftol .and. iscf >= 1) then if (myRank == 1) then write(*,*)""; write(*,*)"SCF converged within",iscf,"steps ..." write(*,*)"SCF error =",scferror endif converged = .true. + exit endif endif enddo @@ -281,7 +287,7 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) real(dp), allocatable :: nguess(:),kernelTimesRes(:) logical, intent(in) :: mix,applyField real(dp) :: tch1 - real(8) :: mls_v, mls_coul, mls_mu, mls_red, mls_mix, mls_scfIter + real(8) :: mls_v, mls_coul, mls_mu, mls_red, mls_mix, mls_scfIter, mls_diag real(dp), allocatable :: KK0Res(:) converged = .false. @@ -341,7 +347,9 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) if(iscf == Nr_SCF) converged = .true. call gpmdcov_msMem("gpmdcov_dm_min_eig", "Before gpmd_diagonalize_H1",lt%verbose,myRank) + if(myRank == 1 .and. lt%verbose >= 1) mls_diag = mls() call gpmdcov_diagonalize_H1(nguess) + call gpmdcov_msI("gpmdcov_DM_Min","Time for diag "//to_string(mls() - mls_diag)//" ms",lt%verbose,myRank) call gpmdcov_msMem("gpmdcov_dm_min_eig", "After gpmd_diagonalize_H1",lt%verbose,myRank) if(lt%MuCalcType == "FromParts" .or. lt%MuCalcType == "Combined")then @@ -356,7 +364,7 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) #endif if(myRank == 1 .and. lt%verbose >= 1) mls_mu = mls() call gpmdcov_muFromParts() - call gpmdcov_msII("gpmdcov_DM_Min","Time for get Mu "//to_string(mls() - mls_mu)//" ms",lt%verbose,myRank) + call gpmdcov_msI("gpmdcov_DM_Min","Time for get Mu "//to_string(mls() - mls_mu)//" ms",lt%verbose,myRank) call gpmdcov_msMem("gpmdcov_dm_min_eig", "After gpmdcov_muFromParts",lt%verbose,myRank) else call gpmdcov_msI("gpmdcov_getmu","No Mu Calculation method. I will use & @@ -542,6 +550,7 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) else !if( kernel%kernelMixing .and. lt%dokernel) call prg_qmixer(nguess,charges_old,dqin,& dqout,scferror,iscf,lt%pulaycoeff,lt%mpulay,0) + scferror = scferror/sqrt(real(sy%nats,dp)) call gpmdcov_msI("gpmdcov_DM_Min","SCF: Doing Pulay/DIIS mixing",lt%verbose,myRank) endif else !if(mix) @@ -604,15 +613,15 @@ subroutine gpmdcov_DM_Min_Eig(Nr_SCF,nguess,mix,applyField) ! call gpmdcov_getKernel_byParts(sy%nats,syprt,syprtk) ! endif ! endif - exit else - if(scferror < lt%scftol .and. iscf > 1) then + if(scferror < lt%scftol .and. iscf >= 1) then if (myRank == 1) then write(*,*)""; write(*,*)"SCF converged within",iscf,"steps ..." write(*,*)"SCF error =",scferror endif converged = .true. + exit endif endif enddo diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 index f0fe0ea5..c560326d 100644 --- a/examples/gpmdk/src/gpmdcov_mdloop.F90 +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -311,7 +311,21 @@ end function cudaProfilerStop endif else call gpmdcov_msMem("gpmdcov_mdloop", "Before prg_xlbo_nint",lt%verbose,myRank) - call prg_xlbo_nint(sy%net_charge,n,n_0,n_1,n_2,n_3,n_4,n_5,mdstep,xl) + + if(gpmdt%xlboon)then + call prg_xlbo_nint(sy%net_charge,n,n_0,n_1,n_2,n_3,n_4,n_5,mdstep,xl) + else + n = sy%net_charge + endif + + if(gpmdt%coarseqmd)then + if(mod(mdstep,gpmdt%finetoleach) == 0)then + lt%scftol = gpmdt%finetol !1.0d-5 + else + lt%scftol = gpmdt%coarsetol !0.01_dp + endif + endif + call gpmdcov_msMem("gpmdcov_mdloop", "After prg_xlbo_nint",lt%verbose,myRank) endif else @@ -376,7 +390,9 @@ end function cudaProfilerStop #ifdef USE_NVTX call nvtxStartRange("InitParts",5) #endif - call gpmdcov_InitParts() + !if((mod(mdstep,lt%nlisteach) == 0 ) .or. (mod(mdstep,gsp2%parteach) == 0) & + ! &.or. mdstep == 0 .or. mdstep == 1) call gpmdcov_InitParts() + call gpmdcov_InitParts() #ifdef USE_NVTX call nvtxEndRange #endif @@ -421,6 +437,8 @@ end function cudaProfilerStop #ifdef USE_NVTX call nvtxStartRange("DM_min",6) #endif + + if(gpmdt%xlboON)then if(eig)then call gpmdcov_msMem("gpmdcov_mdloop", "Before gpmdcov_dm_min",lt%verbose,myRank) call gpmdcov_DM_Min(1,sy%net_charge,.false.) @@ -430,6 +448,7 @@ end function cudaProfilerStop call gpmdcov_DM_Min_Eig(1,sy%net_charge,.false.,.false.) call gpmdcov_msMem("gpmdcov_mdloop", "After gpmdcov_dm_min_Eig",lt%verbose,myRank) endif + endif call gpmdcov_msI("gpmdcov_MDloop","Time for gpmdcov_DM_Min_1 & &"//to_string(mls() - mls_md1)//" ms",lt%verbose,myRank) diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 index 56c5e582..dcb5b075 100644 --- a/examples/gpmdk/src/gpmdcov_parser.F90 +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -132,7 +132,22 @@ module gpmdcov_parser_mod !> Voltage filename character(100) :: voltagef - + + !> XLBO ON/OFF option + logical :: xlboON + + !> Coarse QMD + logical :: coarseqmd + + !> Do fine tol scf each x number of steps + integer :: finetoleach + + !> Fine tol for scf + real(dp) :: finetol + + !> Coarse tol for scf + real(dp) :: coarsetol + end type gpmd_type !> electrontic structure output type @@ -195,7 +210,7 @@ subroutine gpmdcov_parse(filename,gpmdt) implicit none character(len=*), intent(in) :: filename type(gpmd_type), intent(inout) :: gpmdt - integer, parameter :: nkey_char = 5, nkey_int = 9, nkey_re = 7, nkey_log = 13 + integer, parameter :: nkey_char = 5, nkey_int = 10, nkey_re = 9, nkey_log = 15 integer :: i real(dp) :: realtmp character(20) :: dummyc @@ -209,23 +224,23 @@ subroutine gpmdcov_parse(filename,gpmdt) character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & & 'WriteCoordsEach=',"Var2I=","ReplicateX=","ReplicateY=","ReplicateZ=","PartsToTrack=",& - & "DumpEach=","MinimizationSteps=","SMDNumPairs="] + & "DumpEach=","MinimizationSteps=","SMDNumPairs=","FineTolEach="] integer :: valvector_int(nkey_int) = (/ & - & 1, 1, 0, 0, 0, 0, 0, 0, 0/) + & 1, 1, 0, 0, 0, 0, 0, 0, 0, 5/) character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & & 'VRFactor=','InitialTemperature=','LangevinGamma=','SMDForceConstantStart=',& - & 'SMDForceConstantEnd=','SMDR0=','CurrentThreshold='] + & 'SMDForceConstantEnd=','SMDR0=','CurrentThreshold=',"FineTol=","CoarseTol="] real(dp) :: valvector_re(nkey_re) = (/& - & 0.0_dp, 0.0_dp, 0.01_dp, 0.0_dp,0.2_dp,2.0_dp,0.1_dp/) + & 0.0_dp, 0.0_dp, 0.01_dp, 0.0_dp,0.2_dp,2.0_dp,0.1_dp,1.0d-5,0.01_dp/) character(len=50), parameter :: keyvector_log(nkey_log) = [character(len=50) :: & &'DoVelocityRescale=','WriteResidueInTrajectory=','WriteTrajectory=','TrackReactivity=',& &'RestartFromDump=','UseLATTE=','HtoD=','LangevinDynamics=','UseSMD=', & - &'ComputeCurrents=', 'TranslateAndFoldToBox=', 'UseVectSKBlock=', 'ApplyVoltage='] + &'ComputeCurrents=', 'TranslateAndFoldToBox=', 'UseVectSKBlock=', 'ApplyVoltage=','XLBO=','CoarseQMD='] logical :: valvector_log(nkey_log) = (/& &.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false.,.false., & - &.false.,.True.,.false.,.false./) + &.false.,.True.,.false.,.false.,.true.,.false./) !Start and stop characters character(len=50), parameter :: startstop(2) = [character(len=50) :: & @@ -281,6 +296,7 @@ subroutine gpmdcov_parse(filename,gpmdt) gpmdt%dumpeach = valvector_int(7) gpmdt%minimization_steps = valvector_int(8) gpmdt%smdnumpairs = valvector_int(9) + gpmdt%finetoleach = valvector_int(10) if(gpmdt%smdnumpairs > 0) then write(*,*) "Reading SMD pair atom indicies" open(1, file=trim(filename)) @@ -324,6 +340,8 @@ subroutine gpmdcov_parse(filename,gpmdt) gpmdt%smdforceconstend = valvector_re(5) gpmdt%smdr0 = valvector_re(6) gpmdt%currthr = valvector_re(7) + gpmdt%finetol = valvector_re(8) + gpmdt%coarsetol = valvector_re(9) !Logs gpmdt%dovelresc = valvector_log(1) @@ -339,6 +357,8 @@ subroutine gpmdcov_parse(filename,gpmdt) gpmdt%trfl = valvector_log(11) gpmdt%usevectsk = valvector_log(12) gpmdt%applyv = valvector_log(13) + gpmdt%xlboON = valvector_log(14) + gpmdt%coarseqmd = valvector_log(15) if(gpmdt%applyv)then gpmdt%voltagef = valvector_char(5) diff --git a/examples/gpmdk/src/gpmdcov_reshuffle.F90 b/examples/gpmdk/src/gpmdcov_reshuffle.F90 index 649a73cd..6cbc764b 100644 --- a/examples/gpmdk/src/gpmdcov_reshuffle.F90 +++ b/examples/gpmdk/src/gpmdcov_reshuffle.F90 @@ -59,6 +59,11 @@ subroutine gpmdcov_reshuffle() do i=1,getNRanks() costperrank = 0.0d0 do j=1,maxnparts + write(*,*)i,j,reshuffle(j,i) + if(gpat%sgraph(reshuffle(j,i))%lsize == 0)then + call gpmdcov_msI("gpmdcov_reshuffle","!!!ERROR: One part has 0 atoms",lt%verbose,myRank) + stop + endif !if(reshuffle(j,i)>0)write(*,*)i,j,reshuffle(j,i),gpat%sgraph(reshuffle(j,i))%lsize costperrank = costperrank + real((gpat%sgraph(reshuffle(j,i))%lsize)**3) enddo From 7b2ece158addecca0eca2fb630095df538f6821e Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Sun, 15 Dec 2024 13:54:57 -0700 Subject: [PATCH 58/69] Update bml submodule --- bml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bml b/bml index a4dd0321..2e6f60e1 160000 --- a/bml +++ b/bml @@ -1 +1 @@ -Subproject commit a4dd0321ef7df6f227c540327d2c4647cc7ceac0 +Subproject commit 2e6f60e1db7450d4c2f2c4579365138ec889a124 From bfd58e47b16692816eb9c1b764a9b79ac0be8ddb Mon Sep 17 00:00:00 2001 From: rcorrigan Date: Wed, 8 Jan 2025 09:25:37 -0700 Subject: [PATCH 59/69] Added formatting statements to prg_system_mod XYZ trajectory writing code to fix issue with XYZ coordinates being printed across two lines instead of on a single line --- src/prg_system_mod.F90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/prg_system_mod.F90 b/src/prg_system_mod.F90 index 0168f5fb..5a715199 100644 --- a/src/prg_system_mod.F90 +++ b/src/prg_system_mod.F90 @@ -982,12 +982,13 @@ subroutine prg_write_trajectory(system,iter,each,prg_deltat,filename,extension) write(io_unit,*)"frame", iter if(allocated(system%net_charge))then do i=1,nats - write(io_unit,*)system%symbol(i),system%coordinate(1,i),system%coordinate(2,i),system%coordinate(3,i),& - system%net_charge(i) + write(io_unit,"(A3,3X,F20.15,3X,F20.15,3X,F20.15,3X,F20.15)")system%symbol(i),system%coordinate(1,i),& + system%coordinate(2,i),system%coordinate(3,i),system%net_charge(i) enddo else do i=1,nats - write(io_unit,*)system%symbol(i),system%coordinate(1,i),system%coordinate(2,i),system%coordinate(3,i) + write(io_unit,"(A3,3X,F20.15,3X,F20.15,3X,F20.15)")system%symbol(i),system%coordinate(1,i),& + system%coordinate(2,i),system%coordinate(3,i) enddo endif From 8a351167e9b197a6d7cb9c085b9e8b940d328f3f Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Mon, 18 Nov 2024 00:02:06 -0700 Subject: [PATCH 60/69] Opts for nvidia build o Pass subarray of ham, over to get_skblock o Change get_hsmat nested loop to enable collapse(2) In particular, the collapse() clause improves performance --- examples/gpmdk/src/gpmdcov_initparts.F90 | 5 + src/latte_mods/ham_latte_mod.F90 | 192 ++++++++++++++++++++--- 2 files changed, 177 insertions(+), 20 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_initparts.F90 b/examples/gpmdk/src/gpmdcov_initparts.F90 index 869064e1..075b4fe0 100644 --- a/examples/gpmdk/src/gpmdcov_initparts.F90 +++ b/examples/gpmdk/src/gpmdcov_initparts.F90 @@ -74,6 +74,7 @@ subroutine gpmdcov_InitParts call nvtxStartRange("BarrierBeforeGetHS",2) call prg_barrierParallel call nvtxEndRange + call nvtxStartRange("GetHS",2) #endif #endif @@ -98,6 +99,10 @@ subroutine gpmdcov_InitParts &syprt(ipt)%estr%ham0,syprt(ipt)%estr%over) endif +#ifdef USE_NVTX + call nvtxEndRange +#endif + if (myRank == 1 .and. lt%verbose >= 5)then write(*,*)"H0 and S for part:" call bml_print_matrix("H0",syprt(ipt)%estr%ham0,0,6,0,6) diff --git a/src/latte_mods/ham_latte_mod.F90 b/src/latte_mods/ham_latte_mod.F90 index 56f934e6..dff7c9f1 100644 --- a/src/latte_mods/ham_latte_mod.F90 +++ b/src/latte_mods/ham_latte_mod.F90 @@ -14,7 +14,7 @@ module ham_latte_mod integer, parameter :: dp = kind(1.0d0) - public :: get_hindex, get_hindex_coreHalo, get_hsmat, get_hsmat_vect, get_SKBlock, get_SKBlock_vect + public :: get_hindex, get_hindex_coreHalo, get_hsmat, get_hsmat_vect, get_SKBlock, get_SKBlock_vect, get_SKBlock_inplace contains @@ -188,35 +188,36 @@ subroutine get_hsmat(ham_bml,over_bml,coordinate,lattice_vector,spindex,& allocate(blk(maxnorbi,maxnorbi,nats)) endif - !$omp parallel do default(none) & + !$omp parallel do collapse(2) default(none) & !$omp private(i,ra,rb,dimi,dimj,ii,jj,j) & !$omp shared(nats,coordinate,hindex,spindex, intPairsS,intPairsH,threshold,lattice_vector,norbi,onsitesH,onsitesS,ham_bml,over_bml) & !$omp shared(blk,ham,over) do i = 1, nats - ra(:) = coordinate(:,i) - dimi = hindex(2,i)-hindex(1,i)+1 do j = 1, nats + ra(:) = coordinate(:,i) + dimi = hindex(2,i)-hindex(1,i)+1 rb(:) = coordinate(:,j) dimj = hindex(2,j)-hindex(1,j)+1 !Hamiltonian block for a-b atom pair - call get_SKBlock(spindex(i),spindex(j),coordinate(:,i),& + call get_SKBlock_inplace(spindex(i),spindex(j),coordinate(:,i),& coordinate(:,j),lattice_vector,norbi,& - onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,blk,i) - - do jj=1,dimj - do ii=1,dimi - ham(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) - enddo - enddo - - call get_SKBlock(spindex(i),spindex(j),coordinate(:,i),& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams,intPairsH(spindex(j),spindex(i))%intParams,ham(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + !ham(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) = blk(1:dimi,1:dimj,i) + ! do jj=1,dimj + ! do ii=1,dimi + ! ham(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) + ! enddo + ! enddo + + call get_SKBlock_inplace(spindex(i),spindex(j),coordinate(:,i),& coordinate(:,j),lattice_vector,norbi,& - onsitesS,intPairsS(spindex(i),spindex(j))%intParams,intPairsS(spindex(j),spindex(i))%intParams,blk,i) - do jj=1,dimj - do ii=1,dimi - over(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) - enddo - enddo + onsitesS,intPairsS(spindex(i),spindex(j))%intParams,intPairsS(spindex(j),spindex(i))%intParams,over(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + !over(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) = blk(1:dimi,1:dimj,i) + ! do jj=1,dimj + ! do ii=1,dimi + ! over(hindex(1,i)-1+ii,hindex(1,j)-1+jj) = blk(ii,jj,i) + ! enddo + ! enddo ! write(*,*)spindex(i),spindex(j) ! write(*,'(100F10.5)')intPairsS(spindex(i),spindex(j))%intParams @@ -639,6 +640,157 @@ subroutine get_SKBlock(sp1,sp2,coorda,coordb,lattice_vectors& ! stop end subroutine get_SKBlock + !> Standard Slater-Koster sp-parameterization for an atomic block between a pair of atoms + !! \param sp1 Species index for atom 1. This can be obtained from the + !! system type as following: + !! \verbatim sp1 = system%spindex(atom1) \endverbatim + !! \param sp2 Species index for atom 2. + !! \param coorda Coordinates for atom 1. + !! \param coordb Coordinates for atom 2. + !! \param lattice_vectors Lattice vectors for the system. This can be obtained from + !! the system type as following: + !! \verbatim lattice_vectors = system%lattice_vectors \endverbatim + !! \param norbi Number of orbitals for every species in the system. This can be obtained from + !! the tbparams type as following: + !! \verbatim norbi = tbparams%norbi \endverbatim + !! \param onsites Onsites energies for every pair of equal type. Two different variants + !! onsitesH and onsitesS will be used as inputs (see get_hsmat routine) Allocation: + !! \verbatim onsites(maxints,nsp) \endverbatim + !! \param intParams See intpairs_type. + !! \param block Output parameter SK block. + !! \param atnum Input atom number + subroutine get_SKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& + ,norbi,onsites,intParams,intParamsr,blk,atnum) + implicit none + integer :: dimi, dimj, i, nr_shift_X + integer :: nr_shift_Y, nr_shift_Z + integer, intent(in) :: norbi(:), sp1, sp2, atnum + real(dp) :: HPPP, HPPS, HSPS, HSPSR, HSSS + real(dp) :: L, LBox(3), M, N + real(dp) :: PPSMPP, PXPX, PXPY, PXPZ + real(dp) :: PYPX, PYPY, PYPZ, PZPX + real(dp) :: PZPY, PZPZ, dr, ra(3) + real(dp) :: rab(3), rb(3), rxb, ryb + real(dp) :: rzb + real(dp), intent(inout) :: blk(:,:) + real(dp), intent(in) :: coorda(:), coordb(:), intParams(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: onsites(:,:), intParamsr(:,:) + + ra = coorda + rb = coordb + + dimi= norbi(sp1) + dimj= norbi(sp2) + + ! write(*,*)atom_type_a, atom_type_b,dimi,dimj + + !! if(allocated(block))then + !! deallocate(block) + !! endif + + !! allocate(block(dimi,dimj)) + blk(:,:)=0.0_dp + + ! call write_matrix_to_screen("block",block,size(block,dim=1),size(block,dim=2)) + + RXb = Rb(1); RYb = Rb(2); RZb = Rb(3) + + ! For cubic lattice + ! LBox(1) = lattice_vectors(1,1) + ! LBox(2) = lattice_vectors(2,2) + ! LBox(3) = lattice_vectors(3,3) + + !Periodic BC shifts in X, Y and Z. Costs a lot extra! + !do nr_shift_x = -1,1 + ! do nr_shift_y = -1,1 + ! do nr_shift_z = -1,1 + + !rb(1) = RXb + nr_shift_x*lattice_vectors(1,1) ! shifts for pbc + ! rb(1) = rb(1) + nr_shift_y*lattice_vectors(2,1) ! shifts for pbc + ! rb(1) = rb(1) + nr_shift_z*lattice_vectors(3,1) ! shifts for pbc + + !rb(2) = RYb + nr_shift_y*lattice_vectors(2,2) ! shifts for pbc + ! rb(2) = rb(2) + nr_shift_x*lattice_vectors(1,2) ! shifts for pbc + ! rb(2) = rb(2) + nr_shift_z*lattice_vectors(3,2) ! shifts for pbc + + !rb(3) = RZb + nr_shift_z*lattice_vectors(3,3) ! shifts for pbc + ! rb(3) = rb(3) + nr_shift_y*lattice_vectors(2,3) ! shifts for pbc + ! rb(3) = rb(3) + nr_shift_x*lattice_vectors(1,3) ! shifts for pbc + do i = 1,3 + Rab(i) = modulo((Rb(i)-Ra(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) + enddo + !Rab = Rb-Ra; ! OBS b - a !!! + !dR = sqrt(Rab(1)**2+ Rab(2)**2+ Rab(3)**2) + dR = norm2(Rab) + + if(dR.lt.6.5_dp)then + if(dR .LT.1e-12)then !same position and thus the same type sp1 = sp2 + do i=1,dimi + blk(i,i) = onsites(i,sp1) + enddo + else + + L = Rab(1)/dR; !Direction cosines + M = Rab(2)/dR; + N = Rab(3)/dR; + if(dimi == dimj.and.dimi == 1)then !s-s overlap 1 x 1 block + HSSS = BondIntegral(dR,intParams(:,1)) !Calculate the s-s bond integral + blk(1,1) = blk(1,1) + HSSS + elseif(dimi < dimj.and.dimi == 1)then !s-sp overlap 1 x 4 block + HSSS = BondIntegral(dR,intParams(:,1)) + blk(1,1) = blk(1,1) + HSSS + HSPS = BondIntegral(dR,intParams(:,2)) + blk(1,2) = blk(1,2) + L*HSPS + blk(1,3) = blk(1,3) + M*HSPS + blk(1,4) = blk(1,4) + N*HSPS + elseif(dimi > dimj.and.dimj == 1)then ! sp-s overlap 4 x 1 block + HSSS = BondIntegral(dR,intParams(:,1)) + blk(1,1) = blk(1,1) + HSSS + HSPS = BondIntegral(dR,intParams(:,2)) + blk(2,1) = blk(2,1) - L*HSPS + blk(3,1) = blk(3,1) - M*HSPS + blk(4,1) = blk(4,1) - N*HSPS + elseif(dimi == dimj.and.dimj == 4)then !sp-sp overlap + HSSS = BondIntegral(dR,intParams(:,1)) + HSPS = BondIntegral(dR,intParams(:,2)) + HSPSR = BondIntegral(dR,intParamsr(:,2)) + HPPS = BondIntegral(dR,intParams(:,3)) + HPPP = BondIntegral(dR,intParams(:,4)) + PPSMPP = HPPS - HPPP + PXPX = HPPP + L*L*PPSMPP + PXPY = L*M*PPSMPP + PXPZ = L*N*PPSMPP + PYPX = M*L*PPSMPP + PYPY = HPPP + M*M*PPSMPP + PYPZ = M*N*PPSMPP + PZPX = N*L*PPSMPP + PZPY = N*M*PPSMPP + PZPZ = HPPP + N*N*PPSMPP + blk(1,1) = blk(1,1) + HSSS + blk(1,2) = blk(1,2) + L*HSPS + blk(1,3) = blk(1,3) + M*HSPS + blk(1,4) = blk(1,4) + N*HSPS + blk(2,1) = blk(2,1) - L*HSPSR !Change spindex + blk(2,2) = blk(2,2) + PXPX + blk(2,3) = blk(2,3) + PXPY + blk(2,4) = blk(2,4) + PXPZ + blk(3,1) = blk(3,1) - M*HSPSR !Change spindex + blk(3,2) = blk(3,2) + PYPX + blk(3,3) = blk(3,3) + PYPY + blk(3,4) = blk(3,4) + PYPZ + blk(4,1) = blk(4,1) - N*HSPSR !Change spindex + blk(4,2) = blk(4,2) + PZPX + blk(4,3) = blk(4,3) + PZPY + blk(4,4) = blk(4,4) + PZPZ + endif + endif + endif + !enddo + !enddo + !enddo + ! write(*,*)"block",dr,block + ! stop + end subroutine get_SKBlock_inplace !> Standard Slater-Koster sp-parameterization for an atomic block between a pair of atoms !! \param sp1 Species index for atom 1. This can be obtained from the !! system type as following: From 89f9eaa77b6e6b413e84bf332b1740021e0b9ecb Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 19 Nov 2024 13:06:57 -0700 Subject: [PATCH 61/69] NVIDIA opt get_hsmat o Modify get_skblock_inplace to assume zero elements on entry o Modify get_hsmat to zero the ham, over matrices before calc --- src/latte_mods/ham_latte_mod.F90 | 56 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/latte_mods/ham_latte_mod.F90 b/src/latte_mods/ham_latte_mod.F90 index dff7c9f1..c8cf81f3 100644 --- a/src/latte_mods/ham_latte_mod.F90 +++ b/src/latte_mods/ham_latte_mod.F90 @@ -188,6 +188,9 @@ subroutine get_hsmat(ham_bml,over_bml,coordinate,lattice_vector,spindex,& allocate(blk(maxnorbi,maxnorbi,nats)) endif + ham = 0.0_dp + over = 0.0_dp + !$omp parallel do collapse(2) default(none) & !$omp private(i,ra,rb,dimi,dimj,ii,jj,j) & !$omp shared(nats,coordinate,hindex,spindex, intPairsS,intPairsH,threshold,lattice_vector,norbi,onsitesH,onsitesS,ham_bml,over_bml) & @@ -689,7 +692,7 @@ subroutine get_SKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& !! endif !! allocate(block(dimi,dimj)) - blk(:,:)=0.0_dp + !blk(:,:)=0.0_dp ! call write_matrix_to_screen("block",block,size(block,dim=1),size(block,dim=2)) @@ -725,6 +728,7 @@ subroutine get_SKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& if(dR.lt.6.5_dp)then if(dR .LT.1e-12)then !same position and thus the same type sp1 = sp2 +! blk(:,:) = 0.0_dp do i=1,dimi blk(i,i) = onsites(i,sp1) enddo @@ -735,21 +739,21 @@ subroutine get_SKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& N = Rab(3)/dR; if(dimi == dimj.and.dimi == 1)then !s-s overlap 1 x 1 block HSSS = BondIntegral(dR,intParams(:,1)) !Calculate the s-s bond integral - blk(1,1) = blk(1,1) + HSSS + blk(1,1) = + HSSS elseif(dimi < dimj.and.dimi == 1)then !s-sp overlap 1 x 4 block HSSS = BondIntegral(dR,intParams(:,1)) - blk(1,1) = blk(1,1) + HSSS + blk(1,1) = + HSSS HSPS = BondIntegral(dR,intParams(:,2)) - blk(1,2) = blk(1,2) + L*HSPS - blk(1,3) = blk(1,3) + M*HSPS - blk(1,4) = blk(1,4) + N*HSPS + blk(1,2) = + L*HSPS + blk(1,3) = + M*HSPS + blk(1,4) = + N*HSPS elseif(dimi > dimj.and.dimj == 1)then ! sp-s overlap 4 x 1 block HSSS = BondIntegral(dR,intParams(:,1)) - blk(1,1) = blk(1,1) + HSSS + blk(1,1) = + HSSS HSPS = BondIntegral(dR,intParams(:,2)) - blk(2,1) = blk(2,1) - L*HSPS - blk(3,1) = blk(3,1) - M*HSPS - blk(4,1) = blk(4,1) - N*HSPS + blk(2,1) = - L*HSPS + blk(3,1) = - M*HSPS + blk(4,1) = - N*HSPS elseif(dimi == dimj.and.dimj == 4)then !sp-sp overlap HSSS = BondIntegral(dR,intParams(:,1)) HSPS = BondIntegral(dR,intParams(:,2)) @@ -766,22 +770,22 @@ subroutine get_SKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& PZPX = N*L*PPSMPP PZPY = N*M*PPSMPP PZPZ = HPPP + N*N*PPSMPP - blk(1,1) = blk(1,1) + HSSS - blk(1,2) = blk(1,2) + L*HSPS - blk(1,3) = blk(1,3) + M*HSPS - blk(1,4) = blk(1,4) + N*HSPS - blk(2,1) = blk(2,1) - L*HSPSR !Change spindex - blk(2,2) = blk(2,2) + PXPX - blk(2,3) = blk(2,3) + PXPY - blk(2,4) = blk(2,4) + PXPZ - blk(3,1) = blk(3,1) - M*HSPSR !Change spindex - blk(3,2) = blk(3,2) + PYPX - blk(3,3) = blk(3,3) + PYPY - blk(3,4) = blk(3,4) + PYPZ - blk(4,1) = blk(4,1) - N*HSPSR !Change spindex - blk(4,2) = blk(4,2) + PZPX - blk(4,3) = blk(4,3) + PZPY - blk(4,4) = blk(4,4) + PZPZ + blk(1,1) = + HSSS + blk(1,2) = + L*HSPS + blk(1,3) = + M*HSPS + blk(1,4) = + N*HSPS + blk(2,1) = - L*HSPSR !Change spindex + blk(2,2) = + PXPX + blk(2,3) = + PXPY + blk(2,4) = + PXPZ + blk(3,1) = - M*HSPSR !Change spindex + blk(3,2) = + PYPX + blk(3,3) = + PYPY + blk(3,4) = + PYPZ + blk(4,1) = - N*HSPSR !Change spindex + blk(4,2) = + PZPX + blk(4,3) = + PZPY + blk(4,4) = + PZPZ endif endif endif From 305dbc9fde4bab89265cf6644dc4f90ce72e8cf7 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Wed, 20 Nov 2024 11:14:10 -0700 Subject: [PATCH 62/69] NVIDIA opt hsderivative o Create get_dH_or_dS using new get_skblock_inplace method - Initialize matrices with zero o Use new method in gpmdk o Performance is now on par with cray vectorized method --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 13 +- src/latte_mods/hsderivative_latte_mod.F90 | 193 +++++++++++++++++- 2 files changed, 203 insertions(+), 3 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 43b29377..e8849276 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -716,12 +716,21 @@ subroutine gpmdcov_EnergAndForces(charges) &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dSx_bml,dSy_bml,dSz_bml) else - call get_dH(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + ! call get_dH(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + ! &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& + ! &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & + ! <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) + + ! call bml_print_matrix("Old_dH0x_bml",dH0x_bml,0,10,0,10) + + call get_dH_or_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) - call get_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + !call bml_print_matrix("New_dH0x_bml",dH0x_bml,0,10,0,10) + + call get_dH_or_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dSx_bml,dSy_bml,dSz_bml) diff --git a/src/latte_mods/hsderivative_latte_mod.F90 b/src/latte_mods/hsderivative_latte_mod.F90 index 264a23cc..1295686b 100644 --- a/src/latte_mods/hsderivative_latte_mod.F90 +++ b/src/latte_mods/hsderivative_latte_mod.F90 @@ -17,7 +17,7 @@ module hsderivative_latte_mod integer, parameter :: dp = kind(1.0d0) - public :: get_dH, get_dS, get_dH_or_dS_vect + public :: get_dH, get_dS, get_dH_or_dS, get_dH_or_dS_vect contains @@ -233,6 +233,197 @@ subroutine get_dH(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol,lattice_vec ! stop end subroutine get_dH + !> This routine computes the derivative of H matrix. + !! \param dx X differential to compute the derivatives + !! \param coords System coordinates. + !! \param hindex Contains the Hamiltonian indices for every atom (see get_hindex). + !! \param spindex Species indices (see system_type). + !! \param intPairsH See defprg_inition in intPairs_type + !! \param onsitesH Onsite energies for every orbital of a particular species. + !! \param symbol System element symbol. + !! \param lattice_vectors System lattece vectors. + !! \param norb Number of total orbitals. + !! \param norbi Number of orbitals for each atomic site. + !! \param threshold Threshold value for matrix elements. + !! \param dH0x_bml x derivative of H0. + !! \param dH0y_bml y derivative of H0. + !! \param dH0z_bml z derivative of H0. + !! + subroutine get_dH_or_dS(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol,lattice_vectors, norb, norbi, bml_type, & + threshold, dH0x_bml,dH0y_bml,dH0z_bml) + implicit none + character(2) :: Type_pair(2) + character(2), intent(in) :: symbol(:) + character(len=*), intent(in) :: bml_type + integer :: IDim, JDim, nats, dimi + integer :: dimj, i, ii, j + integer :: jj, l + integer, intent(in) :: hindex(:,:), norb, norbi(:), spindex(:) + integer :: maxnorbi + real(dp) :: Rax_m(3), Rax_p(3), Ray_m(3), Ray_p(3) + real(dp) :: Raz_m(3), Raz_p(3), Rb(3), d, maxblockij + real(dp), allocatable :: Rx(:), Ry(:), Rz(:), blockm(:,:,:) + real(dp), allocatable :: dH0x(:,:), dH0y(:,:), dH0z(:,:) + real(dp), allocatable :: H0xm(:,:), H0ym(:,:), H0zm(:,:) + real(dp), intent(in) :: coords(:,:), dx, lattice_vectors(:,:), onsitesH(:,:) + real(dp), intent(in) :: threshold + type(bml_matrix_t), intent(inout) :: dH0x_bml, dH0y_bml, dH0z_bml + type(intpairs_type), intent(in) :: intPairsH(:,:) + ! integer, intent(in) :: nnstruct(:,:) + + + write(*,*)"In get_dH ..." + + nats = size(coords,dim=2) + + if(bml_get_N(dH0x_bml).LT.0)then + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + else + call bml_deallocate(dH0x_bml) + call bml_deallocate(dH0y_bml) + call bml_deallocate(dH0z_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + endif + + ! dH0x = zeros(HDIM,HDIM); dH0y = zeros(HDIM,HDIM); dH0z = zeros(HDIM,HDIM); + + if (.not.allocated(dH0x)) then + allocate(dH0x(norb,norb)) + allocate(dH0y(norb,norb)) + allocate(dH0z(norb,norb)) + allocate(H0xm(norb,norb)) + allocate(H0ym(norb,norb)) + allocate(H0zm(norb,norb)) + endif + + dH0x = 0.0_dp + dH0y = 0.0_dp + dH0z = 0.0_dp + H0xm = 0.0_dp + H0ym = 0.0_dp + H0zm = 0.0_dp + + allocate(Rx(nats)) + allocate(Ry(nats)) + allocate(Rz(nats)) + + Rx = coords(1,:) + Ry = coords(2,:) + Rz = coords(3,:) + + maxnorbi = maxval(norbi) + + !$omp parallel do default(none) private(i) & + !$omp private(Rax_p,Rax_m,Ray_p,Ray_m,Raz_p,Raz_m) & + !$omp private(dimi,J,Type_pair,dimj,Rb,maxblockij) & + !$omp shared(nats,RX,RY,RZ,spindex,hindex,lattice_vectors, dx, threshold) & + !$omp shared(norbi,intPairsH,onsitesH,symbol,dH0x_bml,dH0y_bml,dH0z_bml) & + !$omp shared(dH0x, dH0y, dH0z, H0xm, H0ym, H0zm) + do I = 1, nats + do J = 1,nats + Type_pair(1) = symbol(i); + Rax_p(1) = RX(I)+ dx; Rax_p(2) = RY(I); Rax_p(3) = RZ(I) + Rax_m(1) = RX(I)- dx; Rax_m(2) = RY(I); Rax_m(3) = RZ(I) + Ray_p(1) = RX(I); Ray_p(2) = RY(I)+dx; Ray_p(3) = RZ(I) + Ray_m(1) = RX(I); Ray_m(2) = RY(I)-dx; Ray_m(3) = RZ(I) + Raz_p(1) = RX(I); Raz_p(2) = RY(I); Raz_p(3) = RZ(I)+dx + Raz_m(1) = RX(I); Raz_m(2) = RY(I); Raz_m(3) = RZ(I)-dx + + dimi = hindex(2,I)-hindex(1,I)+1; + if(J .ne. I)then + + Type_pair(2) = symbol(J); + Rb(1) = RX(J); Rb(2) = RY(J); Rb(3) = RZ(J) + dimj = hindex(2,J)-hindex(1,J)+1; + + !! MATLAB code + ! [fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,Es,Ep,U] = LoadBondIntegralParameters_H(Type_pair); % Used in BondIntegral(dR,fxx_xx) + ! diagonal(1:2) = [Es,Ep]; + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + + call get_SKBlock_inplace(spindex(i),spindex(j),Rax_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + if(maxval(abs(dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))) & + .gt.0.0_dp)then + + call get_SKBlock_inplace(spindex(i),spindex(j),Rax_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0xm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Ray_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Ray_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0ym(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Raz_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Raz_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0zm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + endif + endif + enddo + enddo + + !$omp end parallel do + + !call bml_print_matrix("dH0x",dH0x,1,10,1,10) + !call bml_print_matrix("H0xm",H0xm,1,10,1,10) + dH0x = (dH0x - H0xm)/(2.0_dp*dx) + dH0y = (dH0y - H0ym)/(2.0_dp*dx) + dH0z = (dH0z - H0zm)/(2.0_dp*dx) + + call bml_import_from_dense(bml_type,dH0x,dH0x_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dH0y,dH0y_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dH0z,dH0z_bml,threshold,norb) !Dense to dense_bml + + if (allocated(dH0x)) then + deallocate(dH0x) + deallocate(dH0y) + deallocate(dH0z) + deallocate(H0xm) + deallocate(H0ym) + deallocate(H0zm) + endif + + ! stop + end subroutine get_dH_or_dS + !> This routine computes the derivative of S matrix. !! \param dx X differential to compute the derivatives !! \param coords System coordinates. From cb20d3dffefd4c86c4f8a7ed193e464406393878 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Sat, 23 Nov 2024 03:11:02 -0700 Subject: [PATCH 63/69] OpenACC accelerated nonorthocoul o Rewrite CPU code to eliminate packs o Offload the calc for nvidia build --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 229 +++++++++++++++++- examples/gpmdk/src/gpmdcov_mdloop.F90 | 2 +- 2 files changed, 228 insertions(+), 3 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index e8849276..80ef2d32 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -10,7 +10,7 @@ module gpmdcov_EnergAndForces_mod use ham_latte_mod use tbparams_latte_mod - private :: get_dH_or_dS_vect_local, get_skblock_vect_local, bondIntegral_vect_local + private :: get_dH_or_dS_vect_local, get_skblock_vect_local, bondIntegral_vect_local, get_nonortho_coul_forces_local private integer, parameter :: dp = kind(1.0d0) @@ -18,6 +18,231 @@ module gpmdcov_EnergAndForces_mod public :: gpmdcov_EnergAndForces contains + !> Nonortho Coulombic Forces + !! Coulomb force FSCOUL from nonorthogonality + !! \param nats Number of atoms. + !! \param norb Number of orbitals. Its usually the dimension of the Hamiltonian. + !! \param dSx_bml S derivative in the x direction. + !! \param dSy_bml S derivative in the y direction. + !! \param dSz_bml S derivative in the z direction. + !! \param hindex Start and end index for every atom in the system. + !! \param spindex Species index. It gives the species index of a particulat atom. See system_type. + !! \param rho_bml Density matrix in bml format. + !! \param charges Charges for every atom in the system. + !! \param Coulomb_Pot_r Coulombic potential (real space contribution). + !! \param Coulomb_Pot_k Coulombic potential (reciprocal space contribution). + !! \param hubbardu Hubbard parameter U. This is the onsite e-e potential repulsion. it runs over the species list. + !! \param FSCOUL Nonortho coulombinc contribution to the force. + !! \param threshold Threshold value for sparse matrices. + !! + subroutine get_nonortho_coul_forces_local(nats, norb, dSx_bml,dSy_bml,dSz_bml,& + hindex,spindex,rho_bml,charges,Coulomb_Pot_r,Coulomb_Pot_k,hubbardu,FSCOUL,threshold) + implicit none + character(20) :: bml_type + integer :: I_A, I_B, J_A, J_B + integer :: count1, i, j, jj, k + integer :: norb, norbs, nsp + integer, intent(in) :: nats, hindex(:,:), spindex(:) + real(dp) :: dQLxdR, dQLydR, dQLzdR, partrace + real(dp), allocatable :: Coulomb_Pot(:), chunk(:,:), chunkx(:), chunky(:) + real(dp), allocatable :: diagxtmp(:), diagytmp(:), diagztmp(:), row1(:) + real(dp), allocatable :: row2(:), row2x(:), row2y(:), row2z(:) + real(dp), allocatable, intent(inout) :: FSCOUL(:,:) + real(dp), intent(in) :: Coulomb_Pot_k(:), Coulomb_Pot_r(:), charges(:), hubbardu(:) + real(dp), intent(in) :: threshold + type(bml_matrix_t) :: Xtmp_bml, Ytmp_bml, Ztmp_bml, rhot_bml + type(bml_matrix_t), intent(inout) :: dSx_bml, dSy_bml, dSz_bml, rho_bml + real(dp), allocatable :: dSx_dense(:,:), dSy_dense(:,:), dSz_dense(:,:) + real(dp), allocatable :: rho_dense(:,:) + logical, allocatable :: thresh_mask(:,:) + type(c_ptr) :: dSx_bml_c_ptr, dSy_bml_c_ptr, dSz_bml_c_ptr, rho_bml_c_ptr + integer :: ld + real(c_double), pointer :: dSx_bml_ptr(:,:), dSy_bml_ptr(:,:), dSz_bml_ptr(:,:), rho_bml_ptr(:,:) +#ifdef USE_OFFLOAD + real(dp), allocatable :: dDSx(:,:), dDSy(:,:), dDSz(:,:) + +#else + real(dp), allocatable :: dDSx(:), dDSy(:), dDSz(:) +#endif + + write(*,*)"In get_nonortho_coul_forces ..." + + if(.not.allocated(FSCOUL))then + allocate(FSCOUL(3,nats)) + endif + + FSCOUL = 0.0_dp + + allocate(Coulomb_Pot(nats)) + Coulomb_Pot = Coulomb_Pot_r + Coulomb_Pot_k + + norb = bml_get_N(rho_bml) + nsp = size(hubbardu) + +#ifdef USE_OFFLOAD + write(*,*)"DEBUG: Get pointers" + allocate(dDSX(norb,nats)) + allocate(dDSY(norb,nats)) + allocate(dDSZ(norb,nats)) + dDSX = 0.0_dp + dDSY = 0.0_dp + dDSZ = 0.0_dp + + dSx_bml_c_ptr = bml_get_data_ptr_dense(dSx_bml) + dSy_bml_c_ptr = bml_get_data_ptr_dense(dSy_bml) + dSz_bml_c_ptr = bml_get_data_ptr_dense(dSz_bml) + rho_bml_c_ptr = bml_get_data_ptr_dense(rho_bml) + ld = bml_get_ld_dense(dSx_bml) + + write(*,*)"DEBUG: c_f_pointer calls" + call c_f_pointer(dSx_bml_c_ptr,dSx_bml_ptr,shape=[ld,norb]) + call c_f_pointer(dSy_bml_c_ptr,dSy_bml_ptr,shape=[ld,norb]) + call c_f_pointer(dSz_bml_c_ptr,dSz_bml_ptr,shape=[ld,norb]) + call c_f_pointer(rho_bml_c_ptr,rho_bml_ptr,shape=[ld,norb]) + + write(*,*)"DEBUG: acc loop" + !$acc enter data copyin(dDSX(1:norb,1:nats),dDSY(1:norb,1:nats),dDSZ(1:norb,1:nats)) & + !$acc copyin(hindex(1:2,1:nats),FSCOUL(1:3,1:nats),hubbardu(1:nsp)) & + !$acc copyin(spindex(1:nats),charges(1:nats),coulomb_pot(1:nats)) + + !$acc parallel loop deviceptr(dSx_bml_ptr, dSy_bml_ptr, dSz_bml_ptr) & + !$acc deviceptr(rho_bml_ptr) & + !$acc present(dDSX,dDSY,dDSZ,hindex,FSCOUL,hubbardu,spindex) & + !$acc present(charges,coulomb_pot) & + !$acc private(I_A,I_B,j,J_A,J_B,k) & + !$acc private(dQLxdR,dQLydR,dQLzdR) + do I = 1,nats + I_A = hindex(1,I); + I_B = hindex(2,I); + + do j = I_A,I_B + do k = 1,norb + if(abs(rho_bml_ptr(k,j)).gt.threshold)then + dDSX(j,i) = dDSX(j,i) + rho_bml_ptr(k,j)*dSx_bml_ptr(k,j) + dDSY(j,i) = dDSY(j,i) + rho_bml_ptr(k,j)*dSy_bml_ptr(k,j) + dDSZ(j,i) = dDSZ(j,i) + rho_bml_ptr(k,j)*dSz_bml_ptr(k,j) + dDSX(k,i) = dDSX(k,i) + rho_bml_ptr(j,k)*dSx_bml_ptr(k,j) + dDSY(k,i) = dDSY(k,i) + rho_bml_ptr(j,k)*dSy_bml_ptr(k,j) + dDSZ(k,i) = dDSZ(k,i) + rho_bml_ptr(j,k)*dSz_bml_ptr(k,j) + endif + enddo + enddo + do J = 1,nats + J_A = hindex(1,J); + J_B = hindex(2,J); + dQLxdR = 0.0_dp ; dQLydR = 0.0_dp ; dQLzdR = 0.0_dp + do jj=J_A,J_B + dQLxdR = dQLxdR + dDSX(jj,i); + dQLydR = dQLydR + dDSY(jj,i); + dQLzdR = dQLzdR + dDSZ(jj,i); + enddo + FSCOUL(1,I) = FSCOUL(1,I) - & + dQLxdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(2,I) = FSCOUL(2,I) - & + dQLydR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(3,I) = FSCOUL(3,I) - & + dQLzdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + enddo + enddo + !$acc end parallel do + !$acc exit data copyout(FSCOUL(1:3,1:nats)) & + !$acc delete(dDSX(1:norb,1:nats),dDSY(1:norb,1:nats),dDSZ(1:norb,1:nats)) & + !$acc delete(hindex(1:2,1:nats),hubbardu(1:nsp)) & + !$acc delete(spindex(1:nats),charges(1:nats),coulomb_pot(1:nats)) +#else + allocate(dDSX(norb)) + allocate(dDSY(norb)) + allocate(dDSZ(norb)) + + allocate(dSx_dense(norb,norb)) + allocate(dSy_dense(norb,norb)) + allocate(dSz_dense(norb,norb)) + allocate(rho_dense(norb,norb)) + allocate(thresh_mask(norb,norb)) + + call bml_export_to_dense(dSx_bml,dSx_dense) + call bml_export_to_dense(dSy_bml,dSy_dense) + call bml_export_to_dense(dSz_bml,dSz_dense) + call bml_export_to_dense(rho_bml,rho_dense) + + thresh_mask = abs(rho_dense).gt.threshold + + !$omp parallel do default(none) private(i) & + !$omp private(I_A,I_B,j,J_A,J_B,row1,row2,row2x,row2y,row2z,k) & + !$omp private(dDSX,dDSY,dDSZ,dQLxdR,dQLydR,dQLzdR,count1) & + !$omp shared(nats,hindex,norb,rho_dense) & + !$omp shared(dsx_dense,dsy_dense,dsz_dense,thresh_mask) & + !$omp shared(FSCOUL,hubbardu,spindex,charges,coulomb_pot, threshold) + do I = 1,nats + I_A = hindex(1,I); + I_B = hindex(2,I); + + dDSX = 0.0_dp + dDSY = 0.0_dp + dDSZ = 0.0_dp + + do j = I_A,I_B + do k = 1,norb + if(abs(rho_dense(j,k)).gt.threshold)then + dDSX(j) = dDSX(j) + rho_dense(j,k)*dSx_dense(j,k) + dDSY(j) = dDSY(j) + rho_dense(j,k)*dSy_dense(j,k) + dDSZ(j) = dDSZ(j) + rho_dense(j,k)*dSz_dense(j,k) + dDSX(k) = dDSX(k) + rho_dense(k,j)*dSx_dense(j,k) + dDSY(k) = dDSY(k) + rho_dense(k,j)*dSy_dense(j,k) + dDSZ(k) = dDSZ(k) + rho_dense(k,j)*dSz_dense(j,k) + endif + enddo + enddo + + ! do j = I_A,I_B + ! row1 = pack(rho_dense(j,:),thresh_mask(j,:)) + ! row2x = pack(dSx_dense(j,:),thresh_mask(j,:)) + ! row2y = pack(dSy_dense(j,:),thresh_mask(j,:)) + ! row2z = pack(dSz_dense(j,:),thresh_mask(j,:)) + ! dDSX(j) = dDSX(j) + sum(row1(:)*row2x(:)) + ! dDSY(j) = dDSY(j) + sum(row1(:)*row2y(:)) + ! dDSZ(j) = dDSZ(j) + sum(row1(:)*row2z(:)) + ! row1 = pack(rho_dense(:,j),thresh_mask(:,j)) + ! row2x = pack(dSx_dense(j,:),thresh_mask(:,j)) + ! row2y = pack(dSy_dense(j,:),thresh_mask(:,j)) + ! row2z = pack(dSz_dense(j,:),thresh_mask(:,j)) + ! dDSX(:) = dDSX(:) + unpack(row1(:)*row2x(:),thresh_mask(j,:),0.0_dp) + ! dDSY(:) = dDSY(:) + unpack(row1(:)*row2y(:),thresh_mask(j,:),0.0_dp) + ! dDSZ(:) = dDSZ(:) + unpack(row1(:)*row2z(:),thresh_mask(j,:),0.0_dp) + ! enddo + + do J = 1,nats + J_A = hindex(1,J); + J_B = hindex(2,J); + dQLxdR = 0.0_dp ; dQLydR = 0.0_dp ; dQLzdR = 0.0_dp + do jj=J_A,J_B + dQLxdR = dQLxdR + dDSX(jj); + dQLydR = dQLydR + dDSY(jj); + dQLzdR = dQLzdR + dDSZ(jj); + enddo + FSCOUL(1,I) = FSCOUL(1,I) - & + dQLxdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(2,I) = FSCOUL(2,I) - & + dQLydR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(3,I) = FSCOUL(3,I) - & + dQLzdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + enddo + enddo + !$omp end parallel do + + deallocate(thresh_mask) + deallocate(dSx_dense) + deallocate(dSy_dense) + deallocate(dSz_dense) + deallocate(rho_dense) +#endif + write(*,*)"DEBUG: deallocations" + deallocate(dDSX) + deallocate(dDSY) + deallocate(dDSZ) + deallocate(Coulomb_Pot) + end subroutine get_nonortho_coul_forces_local + !> Function to calculate the bond integral for a given distance !! and coefficients set. !! \param dr distance between atoms. @@ -780,7 +1005,7 @@ subroutine gpmdcov_EnergAndForces(charges) ! &,lt%mdim,lt%bml_type,lt%verbose) - call get_nonortho_coul_forces(syprt(ipt)%nats, norb, dSx_bml,dSy_bml,dSz_bml,& + call get_nonortho_coul_forces_local(syprt(ipt)%nats, norb, dSx_bml,dSy_bml,dSz_bml,& syprt(ipt)%estr%hindex,syprt(ipt)%spindex,syprt(ipt)%estr%rho,syprt(ipt)%net_charge,syprt(ipt)%estr%coul_pot_r,& syprt(ipt)%estr%coul_pot_k,tb%hubbardu,syprt(ipt)%estr%FSCOUL,lt%threshold) #ifdef USE_NVTX diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 index c560326d..7ec98f58 100644 --- a/examples/gpmdk/src/gpmdcov_mdloop.F90 +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -90,7 +90,7 @@ end function cudaProfilerStop if (mdstep == 10) then cuda_error = cudaProfilerStart() endif - if (mdstep == 20) then + if (mdstep == 15) then cuda_error = cudaProfilerStop() endif call nvtxStartRange("MD_iter",1) From 33cf90a6356261d968f429e82e1b0773c0de3ca2 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 14 Jan 2025 09:58:05 -0700 Subject: [PATCH 64/69] Add variables to select MD steps for nsys profiling --- examples/gpmdk/src/gpmdcov_mdloop.F90 | 4 ++-- examples/gpmdk/src/gpmdcov_parser.F90 | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_mdloop.F90 b/examples/gpmdk/src/gpmdcov_mdloop.F90 index 7ec98f58..282f7568 100644 --- a/examples/gpmdk/src/gpmdcov_mdloop.F90 +++ b/examples/gpmdk/src/gpmdcov_mdloop.F90 @@ -87,10 +87,10 @@ end function cudaProfilerStop mls_md = mls() #ifdef USE_NVTX - if (mdstep == 10) then + if (mdstep == gpmdt%profile_start_step) then cuda_error = cudaProfilerStart() endif - if (mdstep == 15) then + if (mdstep == gpmdt%profile_stop_step) then cuda_error = cudaProfilerStop() endif call nvtxStartRange("MD_iter",1) diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 index dcb5b075..5ee6b73d 100644 --- a/examples/gpmdk/src/gpmdcov_parser.F90 +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -148,6 +148,12 @@ module gpmdcov_parser_mod !> Coarse tol for scf real(dp) :: coarsetol + !> MD step to start profiling + integer :: profile_start_step + + !> MD step to stop profiling + integer :: profile_stop_step + end type gpmd_type !> electrontic structure output type @@ -210,7 +216,8 @@ subroutine gpmdcov_parse(filename,gpmdt) implicit none character(len=*), intent(in) :: filename type(gpmd_type), intent(inout) :: gpmdt - integer, parameter :: nkey_char = 5, nkey_int = 10, nkey_re = 9, nkey_log = 15 + integer, parameter :: nkey_char = 5, nkey_int = 12, nkey_re = 9, nkey_log = 15 + integer, parameter :: nkey_char = 4, nkey_int = 11, nkey_re = 7, nkey_log = 12 integer :: i real(dp) :: realtmp character(20) :: dummyc @@ -224,9 +231,9 @@ subroutine gpmdcov_parse(filename,gpmdt) character(len=50), parameter :: keyvector_int(nkey_int) = [character(len=50) :: & & 'WriteCoordsEach=',"Var2I=","ReplicateX=","ReplicateY=","ReplicateZ=","PartsToTrack=",& - & "DumpEach=","MinimizationSteps=","SMDNumPairs=","FineTolEach="] + & "DumpEach=","MinimizationSteps=","SMDNumPairs=","FineTolEach=","ProfileStartStep=","ProfileStopStep="] integer :: valvector_int(nkey_int) = (/ & - & 1, 1, 0, 0, 0, 0, 0, 0, 0, 5/) + & 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, -1, -1/) character(len=50), parameter :: keyvector_re(nkey_re) = [character(len=50) :: & & 'VRFactor=','InitialTemperature=','LangevinGamma=','SMDForceConstantStart=',& @@ -331,7 +338,9 @@ subroutine gpmdcov_parse(filename,gpmdt) do i = 1,gpmdt%smdnumpairs write(*,*) "SMD Pairs Atom 1 ",gpmdt%smdatomind1(i)," SMD Pairs Atom 2 ",gpmdt%smdatomind2(i) enddo - + gpmdt%profile_start_step = valvector_int(10) + gpmdt%profile_stop_step = valvector_int(11) + !Reals gpmdt%velresc_fact = valvector_re(1) gpmdt%temp0 = valvector_re(2) From 0933417cfabd6d8cad07f676188e9e8e75b0b8f7 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Tue, 14 Jan 2025 13:53:01 -0700 Subject: [PATCH 65/69] Disable bisection in gpmd. Nonorthocoul opts. --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 2 +- examples/gpmdk/src/gpmdcov_mod.F90 | 27 ++- .../nonorthocoulombforces_latte_mod.F90 | 158 ++++++++++++++---- 3 files changed, 147 insertions(+), 40 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 80ef2d32..260049c6 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -55,10 +55,10 @@ subroutine get_nonortho_coul_forces_local(nats, norb, dSx_bml,dSy_bml,dSz_bml,& real(dp), allocatable :: dSx_dense(:,:), dSy_dense(:,:), dSz_dense(:,:) real(dp), allocatable :: rho_dense(:,:) logical, allocatable :: thresh_mask(:,:) +#ifdef USE_OFFLOAD type(c_ptr) :: dSx_bml_c_ptr, dSy_bml_c_ptr, dSz_bml_c_ptr, rho_bml_c_ptr integer :: ld real(c_double), pointer :: dSx_bml_ptr(:,:), dSy_bml_ptr(:,:), dSz_bml_ptr(:,:), rho_bml_ptr(:,:) -#ifdef USE_OFFLOAD real(dp), allocatable :: dDSx(:,:), dDSy(:,:), dDSz(:,:) #else diff --git a/examples/gpmdk/src/gpmdcov_mod.F90 b/examples/gpmdk/src/gpmdcov_mod.F90 index d6f8b71b..6134ae66 100644 --- a/examples/gpmdk/src/gpmdcov_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_mod.F90 @@ -6,6 +6,9 @@ !! module gpmdcov_mod +#ifdef USE_NVTX + use gpmdcov_nvtx_mod +#endif private @@ -179,14 +182,23 @@ subroutine gpmdcov_muFromParts() nocc = bndfilTotal*real(sy%estr%norbs,dp) err = .false. +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_musearch",2) +#endif mls_mu = mls() - call gpmdcov_musearch(evalsAll,dvalsAll,beta,nocc,10000,10d-10,& - &Ef,HOMO,LUMO,err,myRank,egap_glob,lt%verbose) - if(err .eqv. .true.)then + !call gpmdcov_musearch(evalsAll,dvalsAll,beta,nocc,10000,10d-10,& + ! &Ef,HOMO,LUMO,err,myRank,egap_glob,lt%verbose) + + + !if(err .eqv. .true.)then call gpmdcov_musearch_bisec(evalsAll,dvalsAll,beta,nocc,10d-10,Ef,myRank,egap_glob,lib_mode,err_status,lt%verbose) - endif + !endif +#ifdef USE_NVTX + call nvtxEndRange +#endif + if(err_status) return call gpmdcov_msI("gpmdcov_muFromParts","Chemical potential (Mu or Ef) ="//to_string(Ef),lt%verbose,myRank) @@ -268,11 +280,18 @@ subroutine gpmdcov_musearch(evals, dvals, beta, nocc, maxMuIter, occTol, mu, & occErr = abs(occ-nocc) do i = 1, maxMuIter + +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_fermifunction",3) +#endif call gpmdcov_fermifunction(beta,evals,mu0,fvals) !do j = 1, norbs ! fvals(j) = 1.0_dp/(exp(beta*(evals(j)-mu0))+1.0_dp) !end do +#ifdef USE_NVTX + call nvtxEndRange +#endif occ = 0.0_dp den = 0.0_dp diff --git a/src/latte_mods/nonorthocoulombforces_latte_mod.F90 b/src/latte_mods/nonorthocoulombforces_latte_mod.F90 index b68188cc..9fcf75fa 100644 --- a/src/latte_mods/nonorthocoulombforces_latte_mod.F90 +++ b/src/latte_mods/nonorthocoulombforces_latte_mod.F90 @@ -38,22 +38,30 @@ subroutine get_nonortho_coul_forces(nats, norb, dSx_bml,dSy_bml,dSz_bml,& implicit none character(20) :: bml_type integer :: I_A, I_B, J_A, J_B - integer :: count1, i, j, jj - integer :: norb, norbs + integer :: count1, i, j, jj, k + integer :: norb, norbs, nsp integer, intent(in) :: nats, hindex(:,:), spindex(:) real(dp) :: dQLxdR, dQLydR, dQLzdR, partrace - real(dp), allocatable :: Coulomb_Pot(:), chunk(:,:), chunkx(:,:), chunky(:,:) - real(dp), allocatable :: chunkz(:,:), dDSx(:), dDSy(:), dDSz(:) + real(dp), allocatable :: Coulomb_Pot(:), chunk(:,:), chunkx(:), chunky(:) real(dp), allocatable :: diagxtmp(:), diagytmp(:), diagztmp(:), row1(:) real(dp), allocatable :: row2(:), row2x(:), row2y(:), row2z(:) real(dp), allocatable, intent(inout) :: FSCOUL(:,:) real(dp), intent(in) :: Coulomb_Pot_k(:), Coulomb_Pot_r(:), charges(:), hubbardu(:) real(dp), intent(in) :: threshold type(bml_matrix_t) :: Xtmp_bml, Ytmp_bml, Ztmp_bml, rhot_bml - type(bml_matrix_t), intent(in) :: dSx_bml, dSy_bml, dSz_bml, rho_bml + type(bml_matrix_t), intent(inout) :: dSx_bml, dSy_bml, dSz_bml, rho_bml real(dp), allocatable :: dSx_dense(:,:), dSy_dense(:,:), dSz_dense(:,:) real(dp), allocatable :: rho_dense(:,:) logical, allocatable :: thresh_mask(:,:) +#ifdef USE_OFFLOAD + type(c_ptr) :: dSx_bml_c_ptr, dSy_bml_c_ptr, dSz_bml_c_ptr, rho_bml_c_ptr + integer :: ld + real(c_double), pointer :: dSx_bml_ptr(:,:), dSy_bml_ptr(:,:), dSz_bml_ptr(:,:), rho_bml_ptr(:,:) + real(dp), allocatable :: dDSx(:,:), dDSy(:,:), dDSz(:,:) + +#else + real(dp), allocatable :: dDSx(:), dDSy(:), dDSz(:) +#endif write(*,*)"In get_nonortho_coul_forces ..." @@ -67,7 +75,82 @@ subroutine get_nonortho_coul_forces(nats, norb, dSx_bml,dSy_bml,dSz_bml,& Coulomb_Pot = Coulomb_Pot_r + Coulomb_Pot_k norb = bml_get_N(rho_bml) - bml_type = bml_get_type(dSx_bml) + nsp = size(hubbardu) + +#ifdef USE_OFFLOAD + write(*,*)"DEBUG: Get pointers" + allocate(dDSX(norb,nats)) + allocate(dDSY(norb,nats)) + allocate(dDSZ(norb,nats)) + dDSX = 0.0_dp + dDSY = 0.0_dp + dDSZ = 0.0_dp + + dSx_bml_c_ptr = bml_get_data_ptr_dense(dSx_bml) + dSy_bml_c_ptr = bml_get_data_ptr_dense(dSy_bml) + dSz_bml_c_ptr = bml_get_data_ptr_dense(dSz_bml) + rho_bml_c_ptr = bml_get_data_ptr_dense(rho_bml) + ld = bml_get_ld_dense(dSx_bml) + + write(*,*)"DEBUG: c_f_pointer calls" + call c_f_pointer(dSx_bml_c_ptr,dSx_bml_ptr,shape=[ld,norb]) + call c_f_pointer(dSy_bml_c_ptr,dSy_bml_ptr,shape=[ld,norb]) + call c_f_pointer(dSz_bml_c_ptr,dSz_bml_ptr,shape=[ld,norb]) + call c_f_pointer(rho_bml_c_ptr,rho_bml_ptr,shape=[ld,norb]) + + write(*,*)"DEBUG: acc loop" + !$acc enter data copyin(dDSX(1:norb,1:nats),dDSY(1:norb,1:nats),dDSZ(1:norb,1:nats)) & + !$acc copyin(hindex(1:2,1:nats),FSCOUL(1:3,1:nats),hubbardu(1:nsp)) & + !$acc copyin(spindex(1:nats),charges(1:nats),coulomb_pot(1:nats)) + + !$acc parallel loop deviceptr(dSx_bml_ptr, dSy_bml_ptr, dSz_bml_ptr) & + !$acc deviceptr(rho_bml_ptr) & + !$acc present(dDSX,dDSY,dDSZ,hindex,FSCOUL,hubbardu,spindex) & + !$acc present(charges,coulomb_pot) & + !$acc private(I_A,I_B,j,J_A,J_B,k) & + !$acc private(dQLxdR,dQLydR,dQLzdR) + do I = 1,nats + I_A = hindex(1,I); + I_B = hindex(2,I); + + do j = I_A,I_B + do k = 1,norb + if(abs(rho_bml_ptr(k,j)).gt.threshold)then + dDSX(j,i) = dDSX(j,i) + rho_bml_ptr(k,j)*dSx_bml_ptr(k,j) + dDSY(j,i) = dDSY(j,i) + rho_bml_ptr(k,j)*dSy_bml_ptr(k,j) + dDSZ(j,i) = dDSZ(j,i) + rho_bml_ptr(k,j)*dSz_bml_ptr(k,j) + dDSX(k,i) = dDSX(k,i) + rho_bml_ptr(j,k)*dSx_bml_ptr(k,j) + dDSY(k,i) = dDSY(k,i) + rho_bml_ptr(j,k)*dSy_bml_ptr(k,j) + dDSZ(k,i) = dDSZ(k,i) + rho_bml_ptr(j,k)*dSz_bml_ptr(k,j) + endif + enddo + enddo + do J = 1,nats + J_A = hindex(1,J); + J_B = hindex(2,J); + dQLxdR = 0.0_dp ; dQLydR = 0.0_dp ; dQLzdR = 0.0_dp + do jj=J_A,J_B + dQLxdR = dQLxdR + dDSX(jj,i); + dQLydR = dQLydR + dDSY(jj,i); + dQLzdR = dQLzdR + dDSZ(jj,i); + enddo + FSCOUL(1,I) = FSCOUL(1,I) - & + dQLxdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(2,I) = FSCOUL(2,I) - & + dQLydR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + FSCOUL(3,I) = FSCOUL(3,I) - & + dQLzdR*(hubbardu(spindex(J))*charges(J) + Coulomb_Pot(J)); + enddo + enddo + !$acc end parallel do + !$acc exit data copyout(FSCOUL(1:3,1:nats)) & + !$acc delete(dDSX(1:norb,1:nats),dDSY(1:norb,1:nats),dDSZ(1:norb,1:nats)) & + !$acc delete(hindex(1:2,1:nats),hubbardu(1:nsp)) & + !$acc delete(spindex(1:nats),charges(1:nats),coulomb_pot(1:nats)) +#else + allocate(dDSX(norb)) + allocate(dDSY(norb)) + allocate(dDSZ(norb)) allocate(dSx_dense(norb,norb)) allocate(dSy_dense(norb,norb)) @@ -80,22 +163,14 @@ subroutine get_nonortho_coul_forces(nats, norb, dSx_bml,dSy_bml,dSz_bml,& call bml_export_to_dense(dSz_bml,dSz_dense) call bml_export_to_dense(rho_bml,rho_dense) - allocate(dDSX(norb)) - allocate(dDSY(norb)) - allocate(dDSZ(norb)) - - dDSX = 0.0_dp - dDSY = 0.0_dp - dDSZ = 0.0_dp - thresh_mask = abs(rho_dense).gt.threshold !$omp parallel do default(none) private(i) & - !$omp private(I_A,I_B,j,J_A,J_B,row1,row2,row2x,row2y,row2z,jj) & + !$omp private(I_A,I_B,j,J_A,J_B,row1,row2,row2x,row2y,row2z,k) & !$omp private(dDSX,dDSY,dDSZ,dQLxdR,dQLydR,dQLzdR,count1) & !$omp shared(nats,hindex,norb,rho_dense) & !$omp shared(dsx_dense,dsy_dense,dsz_dense,thresh_mask) & - !$omp shared(FSCOUL,hubbardu,spindex,charges,coulomb_pot) + !$omp shared(FSCOUL,hubbardu,spindex,charges,coulomb_pot, threshold) do I = 1,nats I_A = hindex(1,I); I_B = hindex(2,I); @@ -105,21 +180,34 @@ subroutine get_nonortho_coul_forces(nats, norb, dSx_bml,dSy_bml,dSz_bml,& dDSZ = 0.0_dp do j = I_A,I_B - row1 = pack(rho_dense(j,:),thresh_mask(j,:)) - row2x = pack(dSx_dense(j,:),thresh_mask(j,:)) - row2y = pack(dSy_dense(j,:),thresh_mask(j,:)) - row2z = pack(dSz_dense(j,:),thresh_mask(j,:)) - dDSX(j) = dDSX(j) + sum(row1(:)*row2x(:)) - dDSY(j) = dDSY(j) + sum(row1(:)*row2y(:)) - dDSZ(j) = dDSZ(j) + sum(row1(:)*row2z(:)) - row1 = pack(rho_dense(:,j),thresh_mask(:,j)) - row2x = pack(dSx_dense(j,:),thresh_mask(:,j)) - row2y = pack(dSy_dense(j,:),thresh_mask(:,j)) - row2z = pack(dSz_dense(j,:),thresh_mask(:,j)) - dDSX(:) = dDSX(:) + unpack(row1(:)*row2x(:),thresh_mask(j,:),0.0_dp) - dDSY(:) = dDSY(:) + unpack(row1(:)*row2y(:),thresh_mask(j,:),0.0_dp) - dDSZ(:) = dDSZ(:) + unpack(row1(:)*row2z(:),thresh_mask(j,:),0.0_dp) + do k = 1,norb + if(abs(rho_dense(j,k)).gt.threshold)then + dDSX(j) = dDSX(j) + rho_dense(j,k)*dSx_dense(j,k) + dDSY(j) = dDSY(j) + rho_dense(j,k)*dSy_dense(j,k) + dDSZ(j) = dDSZ(j) + rho_dense(j,k)*dSz_dense(j,k) + dDSX(k) = dDSX(k) + rho_dense(k,j)*dSx_dense(j,k) + dDSY(k) = dDSY(k) + rho_dense(k,j)*dSy_dense(j,k) + dDSZ(k) = dDSZ(k) + rho_dense(k,j)*dSz_dense(j,k) + endif + enddo enddo + + ! do j = I_A,I_B + ! row1 = pack(rho_dense(j,:),thresh_mask(j,:)) + ! row2x = pack(dSx_dense(j,:),thresh_mask(j,:)) + ! row2y = pack(dSy_dense(j,:),thresh_mask(j,:)) + ! row2z = pack(dSz_dense(j,:),thresh_mask(j,:)) + ! dDSX(j) = dDSX(j) + sum(row1(:)*row2x(:)) + ! dDSY(j) = dDSY(j) + sum(row1(:)*row2y(:)) + ! dDSZ(j) = dDSZ(j) + sum(row1(:)*row2z(:)) + ! row1 = pack(rho_dense(:,j),thresh_mask(:,j)) + ! row2x = pack(dSx_dense(j,:),thresh_mask(:,j)) + ! row2y = pack(dSy_dense(j,:),thresh_mask(:,j)) + ! row2z = pack(dSz_dense(j,:),thresh_mask(:,j)) + ! dDSX(:) = dDSX(:) + unpack(row1(:)*row2x(:),thresh_mask(j,:),0.0_dp) + ! dDSY(:) = dDSY(:) + unpack(row1(:)*row2y(:),thresh_mask(j,:),0.0_dp) + ! dDSZ(:) = dDSZ(:) + unpack(row1(:)*row2z(:),thresh_mask(j,:),0.0_dp) + ! enddo do J = 1,nats J_A = hindex(1,J); @@ -140,16 +228,16 @@ subroutine get_nonortho_coul_forces(nats, norb, dSx_bml,dSy_bml,dSz_bml,& enddo !$omp end parallel do - deallocate(dDSX) - deallocate(dDSY) - deallocate(dDSZ) deallocate(thresh_mask) deallocate(dSx_dense) deallocate(dSy_dense) deallocate(dSz_dense) deallocate(rho_dense) - - call bml_deallocate(rhot_bml) +#endif + deallocate(dDSX) + deallocate(dDSY) + deallocate(dDSZ) + deallocate(Coulomb_Pot) end subroutine get_nonortho_coul_forces From 03ef66201826946d02f376c9fc6146e66361a29a Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Thu, 23 Jan 2025 13:32:22 -0700 Subject: [PATCH 66/69] Fix merge error --- examples/gpmdk/src/gpmdcov_parser.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/gpmdk/src/gpmdcov_parser.F90 b/examples/gpmdk/src/gpmdcov_parser.F90 index 5ee6b73d..e908de1e 100644 --- a/examples/gpmdk/src/gpmdcov_parser.F90 +++ b/examples/gpmdk/src/gpmdcov_parser.F90 @@ -217,7 +217,6 @@ subroutine gpmdcov_parse(filename,gpmdt) character(len=*), intent(in) :: filename type(gpmd_type), intent(inout) :: gpmdt integer, parameter :: nkey_char = 5, nkey_int = 12, nkey_re = 9, nkey_log = 15 - integer, parameter :: nkey_char = 4, nkey_int = 11, nkey_re = 7, nkey_log = 12 integer :: i real(dp) :: realtmp character(20) :: dummyc From aa9f7953f726f8ed97d951a2aba75a28ff6d8b9c Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 31 Jan 2025 11:41:00 -0700 Subject: [PATCH 67/69] More NTVX tags --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 417 +++++++++++++++++- examples/gpmdk/src/gpmdcov_lib_mod.F90 | 31 +- examples/gpmdk/src/gpmdcov_nvtx.F90 | 2 +- 3 files changed, 445 insertions(+), 5 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 260049c6..bc1c92cf 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -10,7 +10,7 @@ module gpmdcov_EnergAndForces_mod use ham_latte_mod use tbparams_latte_mod - private :: get_dH_or_dS_vect_local, get_skblock_vect_local, bondIntegral_vect_local, get_nonortho_coul_forces_local + private :: get_dH_or_dS_vect_local, get_skblock_vect_local, get_SKBlock_inplace_local, get_dH_or_dS_local, bondIntegral_vect_local, get_nonortho_coul_forces_local, bondintegral_local private integer, parameter :: dp = kind(1.0d0) @@ -279,6 +279,417 @@ function bondIntegral_vect_local(dr,f) result(x) deallocate(rmod) end function bondIntegral_vect_local + !> Function to calculate the bond integral for a given distance + !! and coefficients set. + !! \param dr distance between atoms. + !! \param f parameters (coefficients) for the bond integral. + real(dp) function bondIntegral_local(dr,f) + implicit none + real(dp) :: rmod + real(dp) :: polynom + real(dp) :: rminusr1 + real(dp) :: x + real(dp), intent(in) :: dr + real(dp), intent(in) :: f(16) + + if(dr <= f(7))then + rmod = dr - f(6); + polynom = rmod*(f(2) + rmod*(f(3) + rmod*(f(4) + f(5)*rmod))); + x = exp(polynom); + elseif(dr > f(7).and.dr < f(8))then + rminusr1 = dr - f(7) + x = f(9) + rminusr1*(f(10) + rminusr1*(f(11) + rminusr1*(f(12) + rminusr1*(f(13) + rminusr1*f(14))))) + else + x = 0 + end if + bondintegral_local = f(1)*x + + end function bondintegral_local + + !> This routine computes the derivative of H matrix. + !! \param dx X differential to compute the derivatives + !! \param coords System coordinates. + !! \param hindex Contains the Hamiltonian indices for every atom (see get_hindex). + !! \param spindex Species indices (see system_type). + !! \param intPairsH See defprg_inition in intPairs_type + !! \param onsitesH Onsite energies for every orbital of a particular species. + !! \param symbol System element symbol. + !! \param lattice_vectors System lattece vectors. + !! \param norb Number of total orbitals. + !! \param norbi Number of orbitals for each atomic site. + !! \param threshold Threshold value for matrix elements. + !! \param dH0x_bml x derivative of H0. + !! \param dH0y_bml y derivative of H0. + !! \param dH0z_bml z derivative of H0. + !! + subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol,lattice_vectors, norb, norbi, bml_type, & + threshold, dH0x_bml,dH0y_bml,dH0z_bml) + implicit none + character(2) :: Type_pair(2) + character(2), intent(in) :: symbol(:) + character(len=*), intent(in) :: bml_type + integer :: IDim, JDim, nats, dimi + integer :: dimj, i, ii, j + integer :: jj, l + integer, intent(in) :: hindex(:,:), norb, norbi(:), spindex(:) + integer :: maxnorbi + real(dp) :: Rax_m(3), Rax_p(3), Ray_m(3), Ray_p(3) + real(dp) :: Raz_m(3), Raz_p(3), Rb(3), d, maxblockij + real(dp), allocatable :: Rx(:), Ry(:), Rz(:), blockm(:,:,:) + real(dp), allocatable :: dH0x(:,:), dH0y(:,:), dH0z(:,:) + real(dp), allocatable :: H0xm(:,:), H0ym(:,:), H0zm(:,:) + real(dp), intent(in) :: coords(:,:), dx, lattice_vectors(:,:), onsitesH(:,:) + real(dp), intent(in) :: threshold + type(bml_matrix_t), intent(inout) :: dH0x_bml, dH0y_bml, dH0z_bml + type(intpairs_type), intent(in) :: intPairsH(:,:) + ! integer, intent(in) :: nnstruct(:,:) + + + write(*,*)"In get_dH ..." + + nats = size(coords,dim=2) + +#ifdef USE_NVTX + call nvtxStartRange("GPUMatrixAllocation",3) +#endif + + if(bml_get_N(dH0x_bml).LT.0)then + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + else + call bml_deallocate(dH0x_bml) + call bml_deallocate(dH0y_bml) + call bml_deallocate(dH0z_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0x_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0y_bml) + call bml_noinit_matrix(bml_type,bml_element_real,dp,norb,norb,dH0z_bml) + endif + +#ifdef USE_NVTX + call nvtxEndRange +#endif + + ! dH0x = zeros(HDIM,HDIM); dH0y = zeros(HDIM,HDIM); dH0z = zeros(HDIM,HDIM); +#ifdef USE_NVTX + call nvtxStartRange("CPUMatrixAllocation",4) +#endif + + if (.not.allocated(dH0x)) then + allocate(dH0x(norb,norb)) + allocate(dH0y(norb,norb)) + allocate(dH0z(norb,norb)) + allocate(H0xm(norb,norb)) + allocate(H0ym(norb,norb)) + allocate(H0zm(norb,norb)) + endif + +#ifdef USE_NVTX + call nvtxEndRange +#endif + +#ifdef USE_NVTX + call nvtxStartRange("CPUMatrixInitialization",5) +#endif + dH0x = 0.0_dp + dH0y = 0.0_dp + dH0z = 0.0_dp + H0xm = 0.0_dp + H0ym = 0.0_dp + H0zm = 0.0_dp + +! !$omp parallel do private(i,j) shared(norb,dH0x,dH0y,dH0z,H0xm,H0ym,H0zm) +! do i=1,norb +! do j=1,norb +! dH0x(i,j) = 0.0_dp +! dH0y(i,j) = 0.0_dp +! dH0z(i,j) = 0.0_dp +! H0xm(i,j) = 0.0_dp +! H0ym(i,j) = 0.0_dp +! H0zm(i,j) = 0.0_dp +! enddo +! enddo +! !$omp end parallel do +#ifdef USE_NVTX + call nvtxEndRange +#endif + + allocate(Rx(nats)) + allocate(Ry(nats)) + allocate(Rz(nats)) + + Rx = coords(1,:) + Ry = coords(2,:) + Rz = coords(3,:) + + maxnorbi = maxval(norbi) + + +#ifdef USE_NVTX + call nvtxStartRange("ComputeMatrixElements",6) +#endif + + !$omp parallel do default(none) private(i) & + !$omp private(Rax_p,Rax_m,Ray_p,Ray_m,Raz_p,Raz_m) & + !$omp private(dimi,J,Type_pair,dimj,Rb,maxblockij) & + !$omp shared(nats,RX,RY,RZ,spindex,hindex,lattice_vectors, dx, threshold) & + !$omp shared(norbi,intPairsH,onsitesH,symbol,dH0x_bml,dH0y_bml,dH0z_bml) & + !$omp shared(dH0x, dH0y, dH0z, H0xm, H0ym, H0zm) + do I = 1, nats + do J = 1,nats + Type_pair(1) = symbol(i); + Rax_p(1) = RX(I)+ dx; Rax_p(2) = RY(I); Rax_p(3) = RZ(I) + Rax_m(1) = RX(I)- dx; Rax_m(2) = RY(I); Rax_m(3) = RZ(I) + Ray_p(1) = RX(I); Ray_p(2) = RY(I)+dx; Ray_p(3) = RZ(I) + Ray_m(1) = RX(I); Ray_m(2) = RY(I)-dx; Ray_m(3) = RZ(I) + Raz_p(1) = RX(I); Raz_p(2) = RY(I); Raz_p(3) = RZ(I)+dx + Raz_m(1) = RX(I); Raz_m(2) = RY(I); Raz_m(3) = RZ(I)-dx + + dimi = hindex(2,I)-hindex(1,I)+1; + if(J .ne. I)then + + Type_pair(2) = symbol(J); + Rb(1) = RX(J); Rb(2) = RY(J); Rb(3) = RZ(J) + dimj = hindex(2,J)-hindex(1,J)+1; + + !! MATLAB code + ! [fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,Es,Ep,U] = LoadBondIntegralParameters_H(Type_pair); % Used in BondIntegral(dR,fxx_xx) + ! diagonal(1:2) = [Es,Ep]; + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Rax_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0x(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Ray_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0y(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_p,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) + dh0/(2*dx); + ! dh0 = Slater_Koster_Block(IDim,JDim,Raz_m,Rb,LBox,Type_pair,fss_sigma,fsp_sigma,fpp_sigma,fpp_pi,diagonal); + ! dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) = dH0z(H_INDEX_START(I):H_INDEX_END(I),H_INDEX_START(J):H_INDEX_END(J)) - dh0/(2*dx); + + + call get_SKBlock_inplace(spindex(i),spindex(j),Rax_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + if(maxval(abs(dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))) & + .gt.0.0_dp)then + + call get_SKBlock_inplace(spindex(i),spindex(j),Rax_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0xm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Ray_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Ray_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0ym(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Raz_p,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + call get_SKBlock_inplace(spindex(i),spindex(j),Raz_m,& + Rb,lattice_vectors,norbi,& + onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & + intPairsH(spindex(j),spindex(i))%intParams, & + H0zm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + + endif + endif + enddo + enddo + + + !$omp end parallel do + +#ifdef USE_NVTX + call nvtxEndRange +#endif + + !call bml_print_matrix("dH0x",dH0x,1,10,1,10) + !call bml_print_matrix("H0xm",H0xm,1,10,1,10) +#ifdef USE_NVTX + call nvtxStartRange("CalculateFiniteDifferences",7) +#endif + + dH0x = (dH0x - H0xm)/(2.0_dp*dx) + dH0y = (dH0y - H0ym)/(2.0_dp*dx) + dH0z = (dH0z - H0zm)/(2.0_dp*dx) + +#ifdef USE_NVTX + call nvtxEndRange +#endif + +#ifdef USE_NVTX + call nvtxStartRange("bml_import_from_dense",8) +#endif + + call bml_import_from_dense(bml_type,dH0x,dH0x_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dH0y,dH0y_bml,threshold,norb) !Dense to dense_bml + call bml_import_from_dense(bml_type,dH0z,dH0z_bml,threshold,norb) !Dense to dense_bml +#ifdef USE_NVTX + call nvtxEndRange +#endif + + if (allocated(dH0x)) then + deallocate(dH0x) + deallocate(dH0y) + deallocate(dH0z) + deallocate(H0xm) + deallocate(H0ym) + deallocate(H0zm) + endif + + ! stop + end subroutine get_dH_or_dS_local + + subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& + ,norbi,onsites,intParams,intParamsr,blk,atnum) + implicit none + integer :: dimi, dimj, i, nr_shift_X + integer :: nr_shift_Y, nr_shift_Z + integer, intent(in) :: norbi(:), sp1, sp2, atnum + real(dp) :: HPPP, HPPS, HSPS, HSPSR, HSSS + real(dp) :: L, LBox(3), M, N + real(dp) :: PPSMPP, PXPX, PXPY, PXPZ + real(dp) :: PYPX, PYPY, PYPZ, PZPX + real(dp) :: PZPY, PZPZ, dr, ra(3) + real(dp) :: rab(3), rb(3), rxb, ryb + real(dp) :: rzb + real(dp), intent(inout) :: blk(:,:) + real(dp), intent(in) :: coorda(:), coordb(:), intParams(:,:), lattice_vectors(:,:) + real(dp), intent(in) :: onsites(:,:), intParamsr(:,:) + + ra = coorda + rb = coordb + + dimi= norbi(sp1) + dimj= norbi(sp2) + + ! write(*,*)atom_type_a, atom_type_b,dimi,dimj + + !! if(allocated(block))then + !! deallocate(block) + !! endif + + !! allocate(block(dimi,dimj)) + !blk(:,:)=0.0_dp + + ! call write_matrix_to_screen("block",block,size(block,dim=1),size(block,dim=2)) + + RXb = Rb(1); RYb = Rb(2); RZb = Rb(3) + + ! For cubic lattice + ! LBox(1) = lattice_vectors(1,1) + ! LBox(2) = lattice_vectors(2,2) + ! LBox(3) = lattice_vectors(3,3) + + !Periodic BC shifts in X, Y and Z. Costs a lot extra! + !do nr_shift_x = -1,1 + ! do nr_shift_y = -1,1 + ! do nr_shift_z = -1,1 + + !rb(1) = RXb + nr_shift_x*lattice_vectors(1,1) ! shifts for pbc + ! rb(1) = rb(1) + nr_shift_y*lattice_vectors(2,1) ! shifts for pbc + ! rb(1) = rb(1) + nr_shift_z*lattice_vectors(3,1) ! shifts for pbc + + !rb(2) = RYb + nr_shift_y*lattice_vectors(2,2) ! shifts for pbc + ! rb(2) = rb(2) + nr_shift_x*lattice_vectors(1,2) ! shifts for pbc + ! rb(2) = rb(2) + nr_shift_z*lattice_vectors(3,2) ! shifts for pbc + + !rb(3) = RZb + nr_shift_z*lattice_vectors(3,3) ! shifts for pbc + ! rb(3) = rb(3) + nr_shift_y*lattice_vectors(2,3) ! shifts for pbc + ! rb(3) = rb(3) + nr_shift_x*lattice_vectors(1,3) ! shifts for pbc + do i = 1,3 + Rab(i) = modulo((Rb(i)-Ra(i) + 0.5_dp*lattice_vectors(i,i)),lattice_vectors(i,i)) - 0.5_dp * lattice_vectors(i,i) + enddo + !Rab = Rb-Ra; ! OBS b - a !!! + !dR = sqrt(Rab(1)**2+ Rab(2)**2+ Rab(3)**2) + dR = norm2(Rab) + + if(dR.lt.6.5_dp)then + if(dR .LT.1e-12)then !same position and thus the same type sp1 = sp2 +! blk(:,:) = 0.0_dp + do i=1,dimi + blk(i,i) = onsites(i,sp1) + enddo + else + + L = Rab(1)/dR; !Direction cosines + M = Rab(2)/dR; + N = Rab(3)/dR; + if(dimi == dimj.and.dimi == 1)then !s-s overlap 1 x 1 block + HSSS = bondintegral_local(dR,intParams(:,1)) !Calculate the s-s bond integral + blk(1,1) = + HSSS + elseif(dimi < dimj.and.dimi == 1)then !s-sp overlap 1 x 4 block + HSSS = bondintegral_local(dR,intParams(:,1)) + blk(1,1) = + HSSS + HSPS = bondintegral_local(dR,intParams(:,2)) + blk(1,2) = + L*HSPS + blk(1,3) = + M*HSPS + blk(1,4) = + N*HSPS + elseif(dimi > dimj.and.dimj == 1)then ! sp-s overlap 4 x 1 block + HSSS = bondintegral_local(dR,intParams(:,1)) + blk(1,1) = + HSSS + HSPS = bondintegral_local(dR,intParams(:,2)) + blk(2,1) = - L*HSPS + blk(3,1) = - M*HSPS + blk(4,1) = - N*HSPS + elseif(dimi == dimj.and.dimj == 4)then !sp-sp overlap + HSSS = bondintegral_local(dR,intParams(:,1)) + HSPS = bondintegral_local(dR,intParams(:,2)) + HSPSR = bondintegral_local(dR,intParamsr(:,2)) + HPPS = bondintegral_local(dR,intParams(:,3)) + HPPP = bondintegral_local(dR,intParams(:,4)) + PPSMPP = HPPS - HPPP + PXPX = HPPP + L*L*PPSMPP + PXPY = L*M*PPSMPP + PXPZ = L*N*PPSMPP + PYPX = M*L*PPSMPP + PYPY = HPPP + M*M*PPSMPP + PYPZ = M*N*PPSMPP + PZPX = N*L*PPSMPP + PZPY = N*M*PPSMPP + PZPZ = HPPP + N*N*PPSMPP + blk(1,1) = + HSSS + blk(1,2) = + L*HSPS + blk(1,3) = + M*HSPS + blk(1,4) = + N*HSPS + blk(2,1) = - L*HSPSR !Change spindex + blk(2,2) = + PXPX + blk(2,3) = + PXPY + blk(2,4) = + PXPZ + blk(3,1) = - M*HSPSR !Change spindex + blk(3,2) = + PYPX + blk(3,3) = + PYPY + blk(3,4) = + PYPZ + blk(4,1) = - N*HSPSR !Change spindex + blk(4,2) = + PZPX + blk(4,3) = + PZPY + blk(4,4) = + PZPZ + endif + endif + endif + !enddo + !enddo + !enddo + ! write(*,*)"block",dr,block + ! stop + end subroutine get_SKBlock_inplace_local !> Standard Slater-Koster sp-parameterization for an atomic block between a pair of atoms !! \param sp1 Species index for atom 1. This can be obtained from the @@ -948,14 +1359,14 @@ subroutine gpmdcov_EnergAndForces(charges) ! call bml_print_matrix("Old_dH0x_bml",dH0x_bml,0,10,0,10) - call get_dH_or_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + call get_dH_or_dS_local(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsH,onsitesH,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dH0x_bml,dH0y_bml,dH0z_bml) !call bml_print_matrix("New_dH0x_bml",dH0x_bml,0,10,0,10) - call get_dH_or_dS(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& + call get_dH_or_dS_local(dx,syprt(ipt)%coordinate,syprt(ipt)%estr%hindex,& &syprt(ipt)%spindex,intPairsS,onsitesS,syprt(ipt)%symbol,& &syprt(ipt)%lattice_vector, norb, tb%norbi, lt%bml_type, & <%threshold, dSx_bml,dSy_bml,dSz_bml) diff --git a/examples/gpmdk/src/gpmdcov_lib_mod.F90 b/examples/gpmdk/src/gpmdcov_lib_mod.F90 index 2d70ddbd..296ae793 100644 --- a/examples/gpmdk/src/gpmdcov_lib_mod.F90 +++ b/examples/gpmdk/src/gpmdcov_lib_mod.F90 @@ -184,17 +184,40 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors !> First SCF loop up to maxscf. if(eig)then +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_DM_Min",4) +#endif call gpmdcov_DM_Min(lt%maxscf,sy%net_charge,.true.) +#ifdef USE_NVTX + call nvtxEndRange +#endif else +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_DM_Min_Eig",4) +#endif call gpmdcov_DM_Min_Eig(lt%maxscf,sy%net_charge,.true.,.false.) +#ifdef USE_NVTX + call nvtxEndRange +#endif endif - +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_EnergAndForces",1) +#endif !> First calculation of energies and forces. call gpmdcov_EnergAndForces(sy%net_charge) !stop +#ifdef USE_NVTX + call nvtxEndRange +#endif else +#ifdef USE_NVTX + call nvtxStartRange("gpmdcov_get_field_forces",2) +#endif call gpmdcov_get_field_forces(tb%norbi,atomic_numbers_in,field_in) +#ifdef USE_NVTX + call nvtxEndRange +#endif endif charges_out(:) = sy%net_charge(:) @@ -204,7 +227,13 @@ subroutine gpmd_compute(coords_in,atomTypes_in,atomic_numbers_in,lattice_vectors if(allocated(dipoleMoment)) deallocate(dipoleMoment) allocate(dipoleMoment(3)) factor = 1.0_dp +#ifdef USE_NVTX + call nvtxStartRange("prg_compute_dipole",3) +#endif call prg_compute_dipole(sy%net_charge,sy%coordinate,dipoleMoment,factor,lt%verbose) +#ifdef USE_NVTX + call nvtxEndRange +#endif dipole_out(:) = dipoleMoment(:) if(lt%stopAt == "gpmdcov_Energ") then diff --git a/examples/gpmdk/src/gpmdcov_nvtx.F90 b/examples/gpmdk/src/gpmdcov_nvtx.F90 index e0520a31..b17bfed5 100644 --- a/examples/gpmdk/src/gpmdcov_nvtx.F90 +++ b/examples/gpmdk/src/gpmdcov_nvtx.F90 @@ -48,7 +48,7 @@ subroutine nvtxStartRange(name,id) type(nvtxEventAttributes):: event tempName=trim(name)//c_null_char - write(*,*)"gpmdcov_nvtx"," Tag = "//name +! write(*,*)"gpmdcov_nvtx"," Tag = "//name if ( .not. present(id)) then call nvtxRangePush(tempName) else From 06f07f52fe4642c1bf4deb3d9fb6df701959c3f8 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 31 Jan 2025 16:10:57 -0700 Subject: [PATCH 68/69] dH optimization --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 99 ++++++++++--------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index bc1c92cf..2ffe8ebc 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -10,7 +10,7 @@ module gpmdcov_EnergAndForces_mod use ham_latte_mod use tbparams_latte_mod - private :: get_dH_or_dS_vect_local, get_skblock_vect_local, get_SKBlock_inplace_local, get_dH_or_dS_local, bondIntegral_vect_local, get_nonortho_coul_forces_local, bondintegral_local + private :: get_dH_or_dS_vect_local, get_skblock_vect_local, get_SKBlock_inplace_local, get_dH_or_dS_local, bondIntegral_vect_local, get_nonortho_coul_forces_local, bondintegral_local, get_dSKBlock_inplace private integer, parameter :: dp = kind(1.0d0) @@ -394,9 +394,9 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol dH0x = 0.0_dp dH0y = 0.0_dp dH0z = 0.0_dp - H0xm = 0.0_dp - H0ym = 0.0_dp - H0zm = 0.0_dp +! H0xm = 0.0_dp +! H0ym = 0.0_dp +! H0zm = 0.0_dp ! !$omp parallel do private(i,j) shared(norb,dH0x,dH0y,dH0z,H0xm,H0ym,H0zm) ! do i=1,norb @@ -478,11 +478,11 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol if(maxval(abs(dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))) & .gt.0.0_dp)then - call get_SKBlock_inplace(spindex(i),spindex(j),Rax_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),Rax_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & - H0xm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) call get_SKBlock_inplace(spindex(i),spindex(j),Ray_p,& Rb,lattice_vectors,norbi,& @@ -490,11 +490,11 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol intPairsH(spindex(j),spindex(i))%intParams, & dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) - call get_SKBlock_inplace(spindex(i),spindex(j),Ray_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),Ray_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & - H0ym(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) call get_SKBlock_inplace(spindex(i),spindex(j),Raz_p,& Rb,lattice_vectors,norbi,& @@ -502,11 +502,11 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol intPairsH(spindex(j),spindex(i))%intParams, & dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) - call get_SKBlock_inplace(spindex(i),spindex(j),Raz_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),Raz_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & - H0zm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) + dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) endif endif @@ -520,19 +520,20 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol call nvtxEndRange #endif - !call bml_print_matrix("dH0x",dH0x,1,10,1,10) - !call bml_print_matrix("H0xm",H0xm,1,10,1,10) -#ifdef USE_NVTX - call nvtxStartRange("CalculateFiniteDifferences",7) -#endif +! #ifdef USE_NVTX +! call nvtxStartRange("CalculateFiniteDifferences",7) +! #endif - dH0x = (dH0x - H0xm)/(2.0_dp*dx) - dH0y = (dH0y - H0ym)/(2.0_dp*dx) - dH0z = (dH0z - H0zm)/(2.0_dp*dx) +! dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) = (dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) - H0xm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))/(2.0_dp*dx) +! dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) = (dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) - H0ym(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))/(2.0_dp*dx) +! dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) = (dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)) - H0zm(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))/(2.0_dp*dx) -#ifdef USE_NVTX - call nvtxEndRange -#endif +! #ifdef USE_NVTX +! call nvtxEndRange +! #endif + + !call bml_print_matrix("dH0x",dH0x,1,10,1,10) + !call bml_print_matrix("H0xm",H0xm,1,10,1,10) #ifdef USE_NVTX call nvtxStartRange("bml_import_from_dense",8) @@ -557,7 +558,7 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol ! stop end subroutine get_dH_or_dS_local - subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& + subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& ,norbi,onsites,intParams,intParamsr,blk,atnum) implicit none integer :: dimi, dimj, i, nr_shift_X @@ -625,7 +626,7 @@ subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& if(dR .LT.1e-12)then !same position and thus the same type sp1 = sp2 ! blk(:,:) = 0.0_dp do i=1,dimi - blk(i,i) = onsites(i,sp1) + blk(i,i) = blk(i,i) - onsites(i,sp1) enddo else @@ -634,21 +635,21 @@ subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& N = Rab(3)/dR; if(dimi == dimj.and.dimi == 1)then !s-s overlap 1 x 1 block HSSS = bondintegral_local(dR,intParams(:,1)) !Calculate the s-s bond integral - blk(1,1) = + HSSS + blk(1,1) = blk(1,1) - HSSS elseif(dimi < dimj.and.dimi == 1)then !s-sp overlap 1 x 4 block HSSS = bondintegral_local(dR,intParams(:,1)) - blk(1,1) = + HSSS + blk(1,1) = blk(1,1) - HSSS HSPS = bondintegral_local(dR,intParams(:,2)) - blk(1,2) = + L*HSPS - blk(1,3) = + M*HSPS - blk(1,4) = + N*HSPS + blk(1,2) = blk(1,2) - L*HSPS + blk(1,3) = blk(1,3) - M*HSPS + blk(1,4) = blk(1,4) - N*HSPS elseif(dimi > dimj.and.dimj == 1)then ! sp-s overlap 4 x 1 block HSSS = bondintegral_local(dR,intParams(:,1)) - blk(1,1) = + HSSS + blk(1,1) = blk(1,1) - HSSS HSPS = bondintegral_local(dR,intParams(:,2)) - blk(2,1) = - L*HSPS - blk(3,1) = - M*HSPS - blk(4,1) = - N*HSPS + blk(2,1) = blk(2,1) + L*HSPS + blk(3,1) = blk(3,1) + M*HSPS + blk(4,1) = blk(4,1) + N*HSPS elseif(dimi == dimj.and.dimj == 4)then !sp-sp overlap HSSS = bondintegral_local(dR,intParams(:,1)) HSPS = bondintegral_local(dR,intParams(:,2)) @@ -665,22 +666,22 @@ subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& PZPX = N*L*PPSMPP PZPY = N*M*PPSMPP PZPZ = HPPP + N*N*PPSMPP - blk(1,1) = + HSSS - blk(1,2) = + L*HSPS - blk(1,3) = + M*HSPS - blk(1,4) = + N*HSPS - blk(2,1) = - L*HSPSR !Change spindex - blk(2,2) = + PXPX - blk(2,3) = + PXPY - blk(2,4) = + PXPZ - blk(3,1) = - M*HSPSR !Change spindex - blk(3,2) = + PYPX - blk(3,3) = + PYPY - blk(3,4) = + PYPZ - blk(4,1) = - N*HSPSR !Change spindex - blk(4,2) = + PZPX - blk(4,3) = + PZPY - blk(4,4) = + PZPZ + blk(1,1) = blk(1,1) - HSSS + blk(1,2) = blk(1,2) - L*HSPS + blk(1,3) = blk(1,3) - M*HSPS + blk(1,4) = blk(1,4) - N*HSPS + blk(2,1) = blk(2,1) + L*HSPSR !Change spindex + blk(2,2) = blk(2,2) - PXPX + blk(2,3) = blk(2,3) - PXPY + blk(2,4) = blk(2,4) - PXPZ + blk(3,1) = blk(3,1) + M*HSPSR !Change spindex + blk(3,2) = blk(3,2) - PYPX + blk(3,3) = blk(3,3) - PYPY + blk(3,4) = blk(3,4) - PYPZ + blk(4,1) = blk(4,1) + N*HSPSR !Change spindex + blk(4,2) = blk(4,2) - PZPX + blk(4,3) = blk(4,3) - PZPY + blk(4,4) = blk(4,4) - PZPZ endif endif endif @@ -689,7 +690,7 @@ subroutine get_SKBlock_inplace_local(sp1,sp2,coorda,coordb,lattice_vectors& !enddo ! write(*,*)"block",dr,block ! stop - end subroutine get_SKBlock_inplace_local + end subroutine get_dSKBlock_inplace !> Standard Slater-Koster sp-parameterization for an atomic block between a pair of atoms !! \param sp1 Species index for atom 1. This can be obtained from the From dc22aa2b87fe83471f97f03d4a7446e7f3b63827 Mon Sep 17 00:00:00 2001 From: Mike Wall Date: Fri, 31 Jan 2025 16:37:00 -0700 Subject: [PATCH 69/69] Bug fix --- examples/gpmdk/src/gpmdcov_energandforces.F90 | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/examples/gpmdk/src/gpmdcov_energandforces.F90 b/examples/gpmdk/src/gpmdcov_energandforces.F90 index 2ffe8ebc..1b6fe2a4 100644 --- a/examples/gpmdk/src/gpmdcov_energandforces.F90 +++ b/examples/gpmdk/src/gpmdcov_energandforces.F90 @@ -478,7 +478,7 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol if(maxval(abs(dH0x(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)))) & .gt.0.0_dp)then - call get_dSKBlock_inplace(spindex(i),spindex(j),Rax_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),dx,Rax_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & @@ -490,7 +490,7 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol intPairsH(spindex(j),spindex(i))%intParams, & dH0y(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) - call get_dSKBlock_inplace(spindex(i),spindex(j),Ray_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),dx,Ray_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & @@ -502,7 +502,7 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol intPairsH(spindex(j),spindex(i))%intParams, & dH0z(hindex(1,i):hindex(2,i),hindex(1,j):hindex(2,j)),i) - call get_dSKBlock_inplace(spindex(i),spindex(j),Raz_m,& + call get_dSKBlock_inplace(spindex(i),spindex(j),dx,Raz_m,& Rb,lattice_vectors,norbi,& onsitesH,intPairsH(spindex(i),spindex(j))%intParams, & intPairsH(spindex(j),spindex(i))%intParams, & @@ -558,7 +558,7 @@ subroutine get_dH_or_dS_local(dx,coords,hindex,spindex,intPairsH,onsitesH,symbol ! stop end subroutine get_dH_or_dS_local - subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& + subroutine get_dSKBlock_inplace(sp1,sp2,dx,coorda,coordb,lattice_vectors& ,norbi,onsites,intParams,intParamsr,blk,atnum) implicit none integer :: dimi, dimj, i, nr_shift_X @@ -570,11 +570,12 @@ subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& real(dp) :: PYPX, PYPY, PYPZ, PZPX real(dp) :: PZPY, PZPZ, dr, ra(3) real(dp) :: rab(3), rb(3), rxb, ryb - real(dp) :: rzb + real(dp) :: rzb, s real(dp), intent(inout) :: blk(:,:) real(dp), intent(in) :: coorda(:), coordb(:), intParams(:,:), lattice_vectors(:,:) - real(dp), intent(in) :: onsites(:,:), intParamsr(:,:) + real(dp), intent(in) :: onsites(:,:), intParamsr(:,:), dx + s = 1.0_dp/2.0_dp/dx ra = coorda rb = coordb @@ -626,7 +627,7 @@ subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& if(dR .LT.1e-12)then !same position and thus the same type sp1 = sp2 ! blk(:,:) = 0.0_dp do i=1,dimi - blk(i,i) = blk(i,i) - onsites(i,sp1) + blk(i,i) = (blk(i,i) - onsites(i,sp1)) * s enddo else @@ -635,21 +636,21 @@ subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& N = Rab(3)/dR; if(dimi == dimj.and.dimi == 1)then !s-s overlap 1 x 1 block HSSS = bondintegral_local(dR,intParams(:,1)) !Calculate the s-s bond integral - blk(1,1) = blk(1,1) - HSSS + blk(1,1) = (blk(1,1) - HSSS) * s elseif(dimi < dimj.and.dimi == 1)then !s-sp overlap 1 x 4 block HSSS = bondintegral_local(dR,intParams(:,1)) - blk(1,1) = blk(1,1) - HSSS + blk(1,1) = (blk(1,1) - HSSS) * s HSPS = bondintegral_local(dR,intParams(:,2)) - blk(1,2) = blk(1,2) - L*HSPS - blk(1,3) = blk(1,3) - M*HSPS - blk(1,4) = blk(1,4) - N*HSPS + blk(1,2) = (blk(1,2) - L*HSPS) * s + blk(1,3) = (blk(1,3) - M*HSPS) * s + blk(1,4) = (blk(1,4) - N*HSPS) * s elseif(dimi > dimj.and.dimj == 1)then ! sp-s overlap 4 x 1 block HSSS = bondintegral_local(dR,intParams(:,1)) - blk(1,1) = blk(1,1) - HSSS + blk(1,1) = (blk(1,1) - HSSS) * s HSPS = bondintegral_local(dR,intParams(:,2)) - blk(2,1) = blk(2,1) + L*HSPS - blk(3,1) = blk(3,1) + M*HSPS - blk(4,1) = blk(4,1) + N*HSPS + blk(2,1) = (blk(2,1) + L*HSPS) * s + blk(3,1) = (blk(3,1) + M*HSPS) * s + blk(4,1) = (blk(4,1) + N*HSPS) * s elseif(dimi == dimj.and.dimj == 4)then !sp-sp overlap HSSS = bondintegral_local(dR,intParams(:,1)) HSPS = bondintegral_local(dR,intParams(:,2)) @@ -666,22 +667,22 @@ subroutine get_dSKBlock_inplace(sp1,sp2,coorda,coordb,lattice_vectors& PZPX = N*L*PPSMPP PZPY = N*M*PPSMPP PZPZ = HPPP + N*N*PPSMPP - blk(1,1) = blk(1,1) - HSSS - blk(1,2) = blk(1,2) - L*HSPS - blk(1,3) = blk(1,3) - M*HSPS - blk(1,4) = blk(1,4) - N*HSPS - blk(2,1) = blk(2,1) + L*HSPSR !Change spindex - blk(2,2) = blk(2,2) - PXPX - blk(2,3) = blk(2,3) - PXPY - blk(2,4) = blk(2,4) - PXPZ - blk(3,1) = blk(3,1) + M*HSPSR !Change spindex - blk(3,2) = blk(3,2) - PYPX - blk(3,3) = blk(3,3) - PYPY - blk(3,4) = blk(3,4) - PYPZ - blk(4,1) = blk(4,1) + N*HSPSR !Change spindex - blk(4,2) = blk(4,2) - PZPX - blk(4,3) = blk(4,3) - PZPY - blk(4,4) = blk(4,4) - PZPZ + blk(1,1) = (blk(1,1) - HSSS) * s + blk(1,2) = (blk(1,2) - L*HSPS) * s + blk(1,3) = (blk(1,3) - M*HSPS) * s + blk(1,4) = (blk(1,4) - N*HSPS) * s + blk(2,1) = (blk(2,1) + L*HSPSR) * s !Change spindex + blk(2,2) = (blk(2,2) - PXPX) * s + blk(2,3) = (blk(2,3) - PXPY) * s + blk(2,4) = (blk(2,4) - PXPZ) * s + blk(3,1) = (blk(3,1) + M*HSPSR) * s !Change spindex + blk(3,2) = (blk(3,2) - PYPX) * s + blk(3,3) = (blk(3,3) - PYPY) * s + blk(3,4) = (blk(3,4) - PYPZ) * s + blk(4,1) = (blk(4,1) + N*HSPSR) * s !Change spindex + blk(4,2) = (blk(4,2) - PZPX) * s + blk(4,3) = (blk(4,3) - PZPY) * s + blk(4,4) = (blk(4,4) - PZPZ) * s endif endif endif