Update to CalibrationErrors 0.6 (#42)

Fixes #39. Fixes #38.

Co-authored-by: David Widmann <[email protected]>

Project.toml

@@ -1,7 +1,7 @@
 name = "CalibrationTests"
 uuid = "2818745e-0823-50c7-bc2d-405ac343d48b"
 authors = ["David Widmann <[email protected]>"]
-version = "0.5.5"
+version = "0.6.0"
 
 [deps]
 CalibrationErrors = "33913031-fe46-5864-950f-100836f47845"
@@ -16,7 +16,7 @@ StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
 StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
 
 [compat]
-CalibrationErrors = "0.5.15"
+CalibrationErrors = "0.6"
 ConsistencyResampling = "0.3"
 HypothesisTests = "0.8, 0.9, 0.10"
 KernelFunctions = "0.10"

src/CalibrationTests.jl

@@ -30,6 +30,4 @@ include("skce/distribution_free.jl")
 
 include("cme.jl")
 
-include("deprecated.jl")
-
 end # module

src/skce/asymptotic_block.jl

@@ -27,7 +27,7 @@ function AsymptoticBlockSKCETest(
     # evaluate U-statistic of the first block
     istart = 1
     iend = blocksize
-    x = UnbiasedSKCE(kernel)(view(predictions, istart:iend), view(targets, istart:iend))
+    x = SKCE(kernel)(view(predictions, istart:iend), view(targets, istart:iend))
 
     # initialize the estimate and the sum of squares
     estimate = x / 1
@@ -38,7 +38,7 @@ function AsymptoticBlockSKCETest(
         # evaluate U-statistic
         istart += blocksize
         iend += blocksize
-        x = UnbiasedSKCE(kernel)(view(predictions, istart:iend), view(targets, istart:iend))
+        x = SKCE(kernel)(view(predictions, istart:iend), view(targets, istart:iend))
 
         # update the estimate
         Δestimate = x - estimate

src/skce/distribution_free.jl

@@ -23,16 +23,17 @@ end
 
 HypothesisTests.default_tail(::DistributionFreeSKCETest) = :right
 
-function HypothesisTests.pvalue(test::DistributionFreeSKCETest{<:BiasedSKCE})
-    s = sqrt(test.n * test.estimate / test.bound) - 1
-    p = exp(-max(s, zero(s))^2 / 2)
-    return p
-end
-
-function HypothesisTests.pvalue(
-    test::DistributionFreeSKCETest{<:Union{UnbiasedSKCE,BlockUnbiasedSKCE}}
-)
-    return exp(-div(test.n, 2) * (test.estimate / test.bound)^2 / 2)
+function HypothesisTests.pvalue(test::DistributionFreeSKCETest)
+    estimator = test.estimator
+    if estimator.unbiased &&
+        (estimator.blocksize === identity || estimator.blocksize isa Integer)
+        return exp(-div(test.n, 2) * (test.estimate / test.bound)^2 / 2)
+    elseif !estimator.unbiased && estimator.blocksize === identity
+        s = sqrt(test.n * test.estimate / test.bound) - 1
+        return exp(-max(s, zero(s))^2 / 2)
+    else
+        error("estimator is not supported")
+    end
 end
 
 HypothesisTests.testname(::DistributionFreeSKCETest) = "Distribution-free SKCE test"

test/Project.toml

@@ -8,7 +8,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
-CalibrationErrors = "0.5.15"
+CalibrationErrors = "0.6"
 Distributions = "0.23, 0.24, 0.25"
 StableRNGs = "1"
 StatsBase = "0.33"

test/binary_trend.jl

@@ -19,9 +19,9 @@
     @testset "Consistency test" begin
         # define estimators
         estimators = (
-            BiasedSKCE(kernel),
-            UnbiasedSKCE(kernel),
-            (BlockUnbiasedSKCE(kernel, b) for b in (2, 10, 50, 100))...,
+            SKCE(kernel),
+            SKCE(kernel; unbiased=false),
+            (SKCE(kernel; blocksize=b) for b in (2, 10, 50, 100))...,
         )
 
         for estimator in estimators
@@ -38,9 +38,9 @@
     @testset "Distribution-free tests" begin
         # define estimators
         estimators = (
-            BiasedSKCE(kernel),
-            UnbiasedSKCE(kernel),
-            (BlockUnbiasedSKCE(kernel, b) for b in (2, 10, 50, 100))...,
+            SKCE(kernel),
+            SKCE(kernel; unbiased=false),
+            (SKCE(kernel; blocksize=b) for b in (2, 10, 50, 100))...,
         )
 
         for estimator in estimators
@@ -51,8 +51,7 @@
             println(test_consistent)
 
             test_only_two = @inferred(DistributionFreeSKCETest(estimator, data_only_two...))
-            @test @inferred(pvalue(test_only_two)) <
-                (estimator isa Union{UnbiasedSKCE,BlockUnbiasedSKCE} ? 0.4 : 1e-6)
+            @test @inferred(pvalue(test_only_two)) < (estimator.unbiased ? 0.4 : 1e-6)
             println(test_only_two)
         end
     end

test/consistency.jl

@@ -40,8 +40,9 @@
         N = 1_000
 
         for blocksize in (2, 5)
-            ce = BlockUnbiasedSKCE(
-                (ExponentialKernel() ∘ ScaleTransform(0.1)) ⊗ WhiteKernel(), blocksize
+            ce = SKCE(
+                (ExponentialKernel() ∘ ScaleTransform(0.1)) ⊗ WhiteKernel();
+                blocksize=blocksize,
             )
 
             for nclasses in (2, 5, 10)

test/runtests.jl

@@ -34,8 +34,4 @@ Random.seed!(1234)
     @testset "Asymptotic CME" begin
         include("cme.jl")
     end
-
-    @testset "Deprecations" begin
-        include("deprecated.jl")
-    end
 end

test/skce/asymptotic.jl

@@ -1,8 +1,8 @@
 @testset "asymptotic.jl" begin
     @testset "estimate, statistic, and kernel matrix" begin
         kernel = (ExponentialKernel() ∘ ScaleTransform(0.1)) ⊗ WhiteKernel()
-        biasedskce = BiasedSKCE(kernel)
-        unbiasedskce = UnbiasedSKCE(kernel)
+        unbiasedskce = SKCE(kernel)
+        biasedskce = SKCE(kernel; unbiased=false)
 
         for nclasses in (2, 10, 100), nsamples in (10, 50, 100)
             # sample predictions and targets

test/skce/asymptotic_block.jl

@@ -14,7 +14,7 @@
                 ]
                 targets_onlyone = ones(Int, length(predictions))
 
-                skce = BlockUnbiasedSKCE(kernel, blocksize)
+                skce = SKCE(kernel; blocksize=blocksize)
 
                 # for both sets of targets
                 for targets in (targets_consistent, targets_onlyone)

test/skce/distribution_free.jl

@@ -17,13 +17,13 @@
         CalibrationTests.uniformbound(kernel) == 3.2 * 2.7
 
         # default bounds for kernel terms
-        CalibrationTests.uniformbound(BlockUnbiasedSKCE(kernel)) == 2 * 3.2 * 2.7
+        CalibrationTests.uniformbound(SKCE(kernel; blocksize=2)) == 2 * 3.2 * 2.7
     end
 
     @testset "estimator and estimates" begin
         kernel = (ExponentialKernel() ∘ ScaleTransform(0.1)) ⊗ WhiteKernel()
 
-        for skce in (BiasedSKCE(kernel), UnbiasedSKCE(kernel), BlockUnbiasedSKCE(kernel))
+        for skce in (SKCE(kernel), SKCE(kernel; unbiased=false), SKCE(kernel; blocksize=2))
             for nclasses in (2, 10, 100), nsamples in (10, 50, 100)
                 # sample predictions and targets
                 dist = Dirichlet(nclasses, 1)
@@ -53,7 +53,7 @@
 
         pvalues_consistent = Vector{Float64}(undef, 100)
 
-        for skce in (BiasedSKCE(kernel), UnbiasedSKCE(kernel), BlockUnbiasedSKCE(kernel))
+        for skce in (SKCE(kernel), SKCE(kernel; unbiased=false), SKCE(kernel; blocksize=2))
             for nclasses in (2, 10)
                 rng = StableRNG(5921)
                 dist = Dirichlet(nclasses, 1)