introspect layout (contig,chunk)

doc

Author: scivision

README.md

@@ -29,14 +29,6 @@ Tested on systems with HDF5 1.8 and 1.10 including:
 
 Currently, Cygwin does not have *Fortran* HDF5 libraries.
 
-## Not yet handled
-
-It's possible to do these things, if there is user need.
-
-* arrays of rank > 7: this has been stubbed in reader_nd.f90, writer_nd.f90. Only the latest compilers support Fortran 2008 arrays up to rank 15.
-* complex64/complex128: this is not natively handled in HDF5. Popular approaches to complex numbers in HDF5 include h5py's using an HDF5 compound datatype.
-* non-default character kind
-
 ## Build
 
 Requirements:
@@ -133,7 +125,7 @@ type(hdf5_file) :: h5f
 * gzip compression may be applied for rank ≥ 2 arrays by setting `comp_lvl` to a value betwen 1 and 9.
   Shuffle filter is automatically applied for better compression
 * string attributes may be applied to any variable at time of writing or later.
-* `chunk_size` option may be set for better compression
+* `chunk_size` and `comp_lvl` options must be set to **enable compression**
 
 `integer, intent(out) :: ierr` is a mandatory parameter. It will be non-zero if error detected.
 This value should be checked, particularly for write operations to avoid missing error conditions.
@@ -198,6 +190,16 @@ call h5f%read('/foo', A)
 call h5f%finalize(ierr)
 ```
 
+### is dataset contiguous or chunked?
+
+Assumed file handle h5f was already initialized, the logical status is inspected:
+
+```fortran
+is_contig = h5f%is_contig('/foo', ierr)
+
+is_chunked = h5f%is_chunked('/foo', ierr)
+```
+
 ### Create group "scope"
 
 ```fortran
@@ -225,3 +227,12 @@ Note the trailing `/` on `/scope/`, that tells the API you are creating a group
 * Using compilers like PGI or Flang may require first compiling the HDF5 library yourself.
 * Intel compiler HDF5 [compile notes](https://www.hdfgroup.org/downloads/hdf5/source-code/)
 * Polymorphic array rank is implemented by explicit code internally. We could have used pointers, but the code is simple enough to avoid the risk associated with explicit array pointers. Also, `select rank` support requires Gfortran-10 or Intel Fortran 2020, so we didn't want to make too-new compiler restriction.
+
+### Missing datatypes
+
+* arrays of rank > 7: this has been stubbed in reader_nd.f90, writer_nd.f90. Only the latest compilers support Fortran 2008 arrays up to rank 15.
+
+The datatypes below are more complex to handle and may see little use due to their downsides.
+
+* complex64/complex128: this is not natively handled in HDF5. There are performance impacts for compound datatypes, thus many choose to just write two datasets, one each for real and imaginary like foo_r and foo_i
+* non-default character kind

src/hdf5_interface.f90

@@ -28,7 +28,8 @@ module h5fortran
 contains
 !> initialize HDF5 file
 procedure, public :: initialize => hdf_initialize, finalize => hdf_finalize, writeattr, &
-  open => hdf_open_group, close => hdf_close_group, shape => hdf_get_shape
+  open => hdf_open_group, close => hdf_close_group, shape => hdf_get_shape, layout => hdf_get_layout, &
+  exist => hdf_check_exist, is_contig => hdf_is_contig, is_chunked => hdf_is_chunked
 
 !> write group or dataset integer/real
 generic, public   :: write => hdf_write_scalar, hdf_write_1d, hdf_write_2d, hdf_write_3d, &
@@ -143,6 +144,31 @@ module subroutine hdf_get_shape(self, dname, dims, ierr)
 integer, intent(out) :: ierr
 end subroutine hdf_get_shape
 
+module integer function hdf_get_layout(self, dname, ierr) result(layout)
+!! H5D_CONTIGUOUS_F, H5D_CHUNKED_F, H5D_VIRTUAL_F, H5D_COMPACT_F
+class(hdf5_file), intent(in)  :: self
+character(*), intent(in)      :: dname
+integer, intent(out) :: ierr
+end function hdf_get_layout
+
+module logical function hdf_check_exist(self, dname, ierr) result(exists)
+class(hdf5_file), intent(in) :: self
+character(*), intent(in) :: dname
+integer, intent(out) :: ierr
+end function hdf_check_exist
+
+module logical function hdf_is_contig(self, dname, ierr)
+class(hdf5_file), intent(in) :: self
+character(*), intent(in) :: dname
+integer, intent(out) :: ierr
+end function hdf_is_contig
+
+module logical function hdf_is_chunked(self, dname, ierr)
+class(hdf5_file), intent(in) :: self
+character(*), intent(in) :: dname
+integer, intent(out) :: ierr
+end function hdf_is_chunked
+
 
 module subroutine hdf_read_scalar(self, dname, value, ierr)
 class(hdf5_file), intent(in)     :: self

src/read.f90

@@ -1,7 +1,8 @@
 !! This submodule is for reading HDF5, via child submodules
 submodule (h5fortran) read
 
-use hdf5, only : h5dopen_f, h5dread_f, h5dclose_f
+use hdf5, only : h5dopen_f, h5dread_f, h5dclose_f, h5dget_create_plist_f, &
+  h5pget_layout_f, H5D_CONTIGUOUS_F, H5D_CHUNKED_F
 use H5LT, only : h5ltget_dataset_info_f, h5ltread_dataset_f, h5ltget_dataset_ndims_f, &
   h5ltread_dataset_string_f, h5ltpath_valid_f
 
@@ -11,25 +12,18 @@
 
 
 module procedure hdf_setup_read
-
 ! module subroutine hdf_setup_read(self, dname, dims, ierr)
 !   class(hdf5_file), intent(in) :: self
 !   character(*), intent(in) :: dname
 !   integer(HSIZE_T), intent(out) :: dims(:)
 !   integer, intent(out) :: ierr
 integer(SIZE_T) :: dsize
 integer :: dtype
-logical :: exists
 
-call h5ltpath_valid_f(self%lid, dname, .true., exists, ierr)
-if (.not.exists .or. ierr /= 0) then
-  write(stderr,*) 'ERROR: ' // dname // ' does not exist in ' // self%filename
-  ierr = -1
-  return
-endif
+if (.not.self%exist(dname, ierr)) return
 
 call h5ltget_dataset_info_f(self%lid, dname, dims, dtype, dsize, ierr)
-if (check(ierr, 'ERROR: open ' // dname // ' read ' // self%filename)) return
+if (check(ierr, 'ERROR: get_dataset_info ' // dname // ' read ' // self%filename)) return
 
 end procedure hdf_setup_read
 
@@ -38,14 +32,8 @@
 !! must get dims before info, as "dims" must be allocated or segfault occurs.
 integer(SIZE_T) :: dsize
 integer :: dtype, drank
-logical :: exists
 
-call h5ltpath_valid_f(self%lid, dname, .true., exists, ierr)
-if (.not.exists .or. ierr /= 0) then
-  write(stderr,*) 'ERROR: ' // dname // ' does not exist in ' // self%filename
-  ierr = -1
-  return
-endif
+if (.not.self%exist(dname, ierr)) return
 
 call h5ltget_dataset_ndims_f(self%lid, dname, drank, ierr)
 if (check(ierr, 'ERROR: '// dname // ' rank ' // self%filename)) return
@@ -57,4 +45,43 @@
 end procedure hdf_get_shape
 
 
+module procedure hdf_get_layout
+
+integer(HID_T) :: pid, did
+
+layout = -1
+
+if (.not.self%exist(dname, ierr)) return
+
+call h5dopen_f(self%lid, dname, did, ierr)
+if (check(ierr, 'ERROR: open dataset ' // dname // ' read layout ' // self%filename)) return
+call h5dget_create_plist_f(did, pid, ierr)
+if (check(ierr, 'ERROR: get property list ID ' // dname // ' read layout ' // self%filename)) return
+call h5pget_layout_f(pid, layout, ierr)
+if (check(ierr, 'ERROR: get_layout ' // dname // ' read layout ' // self%filename)) return
+call h5dclose_f(did, ierr)
+if (check(ierr, 'ERROR: close dataset: ' // self%filename)) return
+
+end procedure hdf_get_layout
+
+
+module procedure hdf_is_contig
+hdf_is_contig = self%layout(dname, ierr) == H5D_CONTIGUOUS_F
+end procedure hdf_is_contig
+
+module procedure hdf_is_chunked
+hdf_is_chunked = self%layout(dname, ierr) == H5D_CHUNKED_F
+end procedure hdf_is_chunked
+
+
+module procedure hdf_check_exist
+
+call h5ltpath_valid_f(self%lid, dname, .true., exists, ierr)
+if (.not.exists .or. ierr /= 0) then
+  write(stderr,*) 'ERROR: ' // dname // ' does not exist in ' // self%filename
+  ierr = -1
+endif
+
+end procedure hdf_check_exist
+
 end submodule read

src/tests/test_deflate.f90

@@ -2,6 +2,7 @@
 use, intrinsic:: iso_fortran_env, only: int64, int32, real32, real64, stderr=>error_unit
 use, intrinsic:: iso_c_binding, only: c_null_char
 use h5fortran, only: hdf5_file, toLower, strip_trailing_null, truncate_string_null
+use hdf5, only: H5D_CHUNKED_F, H5D_CONTIGUOUS_F
 
 implicit none
 
@@ -29,7 +30,9 @@ subroutine test_hdf5_deflate(path)
 character(*), intent(in) :: path
 integer(int64), parameter :: N=1000
 integer(int64) :: crat, chunk_size(7)
-integer ::  fsize, ibig2(N,N) = 0, ibig3(N,N,4) = 0
+integer ::  fsize, layout
+
+integer :: ibig2(N,N) = 0, ibig3(N,N,4) = 0
 real(real32) :: big2(N,N) = 0., big3(N,N,4) = 0.
 character(:), allocatable :: fn
 
@@ -38,19 +41,35 @@ subroutine test_hdf5_deflate(path)
 
 fn = path // '/deflate1.h5'
 call h5f%initialize(fn, ierr, status='new', action='rw', comp_lvl=1, chunk_size=chunk_size)
-if(ierr/=0) error stop '#1 init'
+if(ierr/=0) error stop '#1 write init'
 call h5f%write('/big2', big2, ierr, chunk_size=[100,100])
-if(ierr/=0) error stop '#1 write'
+if(ierr/=0) error stop '#1 write chunked'
+call h5f%write('/small_contig', big2(:5,:5), ierr, chunk_size=[-1,-1])
+if(ierr/=0) error stop '#1 write contig override'
 call h5f%finalize(ierr)
-if (ierr /= 0) error stop '#1 finalize'
+if (ierr /= 0) error stop '#1 write finalize'
 
 inquire(file=fn, size=fsize)
 crat = (N*N*storage_size(big2)/8) / fsize
-
 print '(A,F6.2,A,I6)','filesize (Mbytes): ',fsize/1e6, '   2D compression ratio:',crat
-
 if (h5f%comp_lvl > 0 .and. crat < 10) error stop '2D low compression'
 
+call h5f%initialize(fn, ierr, status='old', action='r')
+if(ierr/=0) error stop '#1 read init'
+
+layout = h5f%layout('/big2', ierr)
+if(ierr/=0) error stop '#1 get layout chunk'
+if(layout /= H5D_CHUNKED_F) error stop '#1 not chunked layout'
+if(.not.h5f%is_chunked('/big2', ierr)) error stop '#1 not chunked layout'
+
+layout = h5f%layout('/small_contig', ierr)
+if(ierr/=0) error stop '#1 get layout contig'
+if(layout /= H5D_CONTIGUOUS_F) error stop '#1 not contiguous layout'
+if(.not.h5f%is_contig('/small_contig', ierr)) error stop '#1 not contig layout'
+
+call h5f%finalize(ierr)
+if (ierr /= 0) error stop '#1 read finalize'
+
 !======================================
 fn = path // '/deflate2.h5'
 call h5f%initialize(fn, ierr, status='new',action='rw',comp_lvl=1)