Add SignChecker

Author: ph-kev

docs/src/ensemble_builder.md

@@ -157,6 +157,10 @@ of the metadata. For example, the short names are checked between the
 - [`SequentialIndicesChecker`](@ref ClimaCalibrate.Checker.SequentialIndicesChecker):
   Check the indices of the dates of the simulation data corresponding to the
   dates of the metadata is sequential.
+- [`SignChecker`](@ref ClimaCalibrate.Checker.SignChecker):
+  Check that the proportion of positive values in the simulation data and
+  observational data are approximately equal (within a default threshold of
+  0.05).
 
 By default, `GEnsembleBuilder` uses the first five checkers to validate
 compatibility between . You can also provide additional checkers using the `checkers`

ext/checkers.jl

@@ -5,7 +5,8 @@ import ClimaCalibrate.Checker:
     DimUnitsChecker,
     UnitsChecker,
     DimValuesChecker,
-    SequentialIndicesChecker
+    SequentialIndicesChecker,
+    SignChecker
 import ClimaCalibrate.Checker
 
 """
@@ -237,3 +238,37 @@ function Checker.check(
     end
     return true
 end
+
+"""
+    Checker.check(
+        ::SignChecker,
+        var::OutputVar,
+        metadata::Metadata;
+        data,
+        verbose = false,
+    )
+
+Return `true` if the absolute difference of the proportion of positive values in
+`var.data` and the proportion of positive values in `data` is less than the threshold
+defined in `SignChecker`, `false` otherwise.
+"""
+function Checker.check(
+    checker::SignChecker,
+    var::OutputVar,
+    metadata::Metadata;
+    data,
+    verbose = false,
+)
+    obs_pos_proportion = mean(data .> 0)
+
+    # This is inaccurate, because not all the values in var.data will end up in
+    # the G ensemble matrix. See _match_dates for one case. However, the mean
+    # should not change that much with additional times.
+    sim_pos_proportion = nanmean(var.data .> 0)
+
+    same_sign = abs(obs_pos_proportion - sim_pos_proportion) < checker.threshold
+    !same_sign &&
+        verbose &&
+        @info "Proportion of positive values in the simulation data ($sim_pos_proportion) is not the same as the proportion of positive values in the observational data ($obs_pos_proportion)"
+    return same_sign
+end

src/checkers.jl

@@ -77,13 +77,25 @@ corresponding to the dates of the metadata is sequential.
 struct SequentialIndicesChecker <: AbstractChecker end
 
 """
+    struct SignChecker{FT <: AbstractFloat} <: AbstractChecker
+
 A struct that checks that the proportion of positive values in the simulation
 data and observational data is roughly the same.
-"""
-struct SignChecker <: AbstractChecker end
 
+To change the default threshold of 0.05, you can pass a float to `SignChecker`.
+```julia
+import ClimaCalibrate
+sign_checker = ClimaCalibrate.Checker.SignChecker(0.01)
+```
 """
-    check(checker::AbstractChecker; verbose = false)
+@kwdef struct SignChecker{FT <: AbstractFloat} <: AbstractChecker
+    threshold::FT = 0.05
+    function SignChecker(threshold)
+        zero(threshold) <= threshold <= one(threshold) ?
+        new{typeof(threshold)}(threshold) :
+        error("Threshold ($threshold) should be between zero and one")
+    end
+end
 
 """
     check(checker::AbstractChecker,

test/ensemble_builder.jl

@@ -35,6 +35,7 @@ import ClimaAnalysis.Template:
 
 @testset "Is compatible with metadata" begin
     make_metadata(var) = ClimaAnalysis.flatten(var).metadata
+    make_flat_data(var) = ClimaAnalysis.flatten(var).data
     lat = [-90.0, 0.0, 90.0]
     var =
         TemplateVar() |>
@@ -185,6 +186,33 @@ import ClimaAnalysis.Template:
         make_metadata(date_var4),
         verbose = true,
     )
+
+    # Check sign of data
+    sign_checker = Checker.SignChecker(0.05)
+    neg_var = ClimaAnalysis.remake(var, data = -var.data)
+
+    @test Checker.check(
+        sign_checker,
+        var,
+        make_metadata(var),
+        data = make_flat_data(var),
+    )
+    @test !Checker.check(
+        sign_checker,
+        var,
+        make_metadata(neg_var),
+        data = make_flat_data(neg_var),
+    )
+
+    @test_logs (:info, r"Proportion of positive values in the simulation data ") Checker.check(
+        sign_checker,
+        var,
+        make_metadata(neg_var),
+        data = make_flat_data(neg_var),
+        verbose = true,
+    )
+
+    @test_throws ErrorException Checker.SignChecker(-0.05)
 end
 
 @testset "Match dates" begin