Merge pull request #23 from Alexander-Barth/main

allow to use a different masking value than missing

Author: tcarion

Project.toml

@@ -11,7 +11,7 @@ DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
 GRIB = "b16dfd50-4035-11e9-28d4-9dfe17e6779b"
 
 [compat]
-CommonDataModel = "^0.2.1, 0.3"
+CommonDataModel = "0.3.4"
 DataStructures = "0.18"
 DiskArrays = "0.3"
 GRIB = "0.3, 0.4"

src/GRIBDatasets.jl

@@ -5,7 +5,8 @@ using GRIB
 using DataStructures
 import DiskArrays as DA
 using CommonDataModel: AbstractDataset, AbstractVariable, show_dim, CFVariable
-import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib, dataset
+import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib,
+    dataset, maskingvalue
 import CommonDataModel as CDM
 
 const DEFAULT_EPOCH = DateTime(1970, 1, 1, 0, 0)

src/cfvariables.jl

@@ -1,13 +1,14 @@
 
 _get_dim(cfvar::CFVariable, dimname) = _get_dim(cfvar.var, dimname)
-function cfvariable(ds, varname)
+function cfvariable(ds, varname; maskingvalue = maskingvalue(ds))
     v = Variable(ds, string(varname))
     misval = missing_value(v)
     CDM.cfvariable(
         ds, varname;
         _v = v,
         missing_value = isnothing(misval) ? eltype(v)[] : [misval],
         attrib = cflayer_attributes(v),
+        maskingvalue = maskingvalue,
     )
 end
 

src/dataset.jl

@@ -8,19 +8,21 @@ It can be created with the path to the GRIB file:
 ds = GRIBDataset(example_file);
 ```
 """
-struct GRIBDataset{T, N} <: AbstractDataset
+struct GRIBDataset{T, N, Tmaskingvalue} <: AbstractDataset
     index::FileIndex{T}
     dims::NTuple{N, AbstractDim}
     attrib::Dict{String, Any}
+    maskingvalue::Tmaskingvalue
 end
 
 const Dataset = GRIBDataset
 
-function GRIBDataset(index::FileIndex)
-    GRIBDataset(index, _alldims(index), dataset_attributes(index)) 
+function GRIBDataset(index::FileIndex; maskingvalue = missing)
+    GRIBDataset(index, _alldims(index), dataset_attributes(index), maskingvalue)
 end
 
-GRIBDataset(filepath::AbstractString; filter_by_values = Dict()) = GRIBDataset(FileIndex(filepath; filter_by_values))
+GRIBDataset(filepath::AbstractString; filter_by_values = Dict(), kwargs...) =
+    GRIBDataset(FileIndex(filepath; filter_by_values); kwargs...)
 
 Base.keys(ds::Dataset) = getvars(ds)
 Base.haskey(ds::Dataset, key) = key in keys(ds)
@@ -46,6 +48,7 @@ dimnames(ds::GRIBDataset) = keys(ds.dims)
 
 attribnames(ds::GRIBDataset) = keys(ds.attrib)
 attrib(ds::GRIBDataset, attribname::String) = ds.attrib[attribname]
+maskingvalue(ds::GRIBDataset) = ds.maskingvalue
 
 # _dim_values(ds::GRIBDataset, dim::Dimension{Horizontal}) = _dim_values(ds.index, dim)
 

test/dataset.jl

@@ -12,6 +12,7 @@ using GRIBDatasets: CDM
     grib_path = joinpath(dir_testfiles, "era5-levels-members.grib")
     ds = GRIBDataset(grib_path)
     dsmis = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib"))
+    dsNaN = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib"),maskingvalue = NaN)
     index = ds.index
 
     varstring = "z"
@@ -87,12 +88,25 @@ using GRIBDatasets: CDM
         @testset "cfvariable and missing" begin
             cfvar = cfvariable(ds, varstring)
             cfvarmis = cfvariable(dsmis, "t2m")
+            cfvarNaN = cfvariable(dsNaN, "t2m")
             A = cfvar[:,:,1,1,1]
             Amis = cfvarmis[:,:,1,1]
+            ANaN = cfvarNaN[:,:,1,1]
 
             # With CommonDataModel, we necessarily get a Union{Missing, Float64}, even if there's no missing.
             @test_broken eltype(A) == Float64
             @test eltype(Amis) == Union{Missing, Float64}
+            @test eltype(ANaN) == Float64
+
+            # test the use of a different maskingvalue per dataset
+            @test eltype(dsmis["t2m"]) == Union{Missing,Float64}
+            @test eltype(dsNaN["t2m"]) == Float64
+            @test ismissing(Amis[1,1,1])
+            @test isnan(ANaN[1,1,1])
+
+            # test the use of a different maskingvalue per variable
+            A2NaN = cfvariable(dsmis,"t2m", maskingvalue = NaN)
+            @test isnan(A2NaN[1,1,1])
         end
 
         @testset "cfvariable coordinate" begin
@@ -181,4 +195,4 @@ end
         @time ds = GRIBDataset(testfile)
     end
 
-end
+end