Rb/callmip phase1a

.buildkite/Manifest-v1.12.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.12.5"
 manifest_format = "2.0"
-project_hash = "6a0d58b5777a108d85781cd56892da71dd1d06ab"
+project_hash = "3a670ec1f3228eaa501f77111b052a480a84b388"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "f7304359109c768cf32dc5fa2d371565bb63b68a"
@@ -266,9 +266,9 @@ version = "0.1.1"
 
 [[deps.BenchmarkTools]]
 deps = ["Compat", "JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
-git-tree-sha1 = "6876e30dc02dc69f0613cb6ece242144f2ca9e56"
+git-tree-sha1 = "7fecfb1123b8d0232218e2da0c213004ff15358d"
 uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-version = "1.7.0"
+version = "1.6.3"
 
 [[deps.BlockArrays]]
 deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra"]
@@ -375,10 +375,10 @@ uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 version = "0.15.9"
 
 [[deps.Cairo_jll]]
-deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "d0efe2c6fdcdaa1c161d206aa8b933788397ec71"
+deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "a21c5464519504e41e0cbc91f0188e8ca23d7440"
 uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
-version = "1.18.6+0"
+version = "1.18.5+1"
 
 [[deps.ChainRules]]
 deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"]
@@ -388,9 +388,9 @@ version = "1.73.0"
 
 [[deps.ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra"]
-git-tree-sha1 = "12177ad6b3cad7fd50c8b3825ce24a99ad61c18f"
+git-tree-sha1 = "e4c6a16e77171a5f5e25e9646617ab1c276c5607"
 uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-version = "1.26.1"
+version = "1.26.0"
 weakdeps = ["SparseArrays"]
 
     [deps.ChainRulesCore.extensions]
@@ -426,9 +426,9 @@ weakdeps = ["GeoMakie", "Makie"]
 
 [[deps.ClimaCalibrate]]
 deps = ["Dates", "Distributed", "Distributions", "EnsembleKalmanProcesses", "JLD2", "Logging", "Random", "TOML", "YAML"]
-git-tree-sha1 = "614be4383a0caa9214fd47e28c086cc4e8d536b9"
+git-tree-sha1 = "0d7ff225f8cfe2f6adad34ed7ed6f32438d4a772"
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
-version = "0.2.2"
+version = "0.1.4"
 
     [deps.ClimaCalibrate.extensions]
     CESExt = "CalibrateEmulateSample"
@@ -486,10 +486,10 @@ weakdeps = ["CUDA"]
     ClimaInterpolationsCUDAExt = "CUDA"
 
 [[deps.ClimaLand]]
-deps = ["ClimaComms", "ClimaCore", "ClimaDiagnostics", "ClimaParams", "ClimaTimeSteppers", "ClimaUtilities", "Dates", "DocStringExtensions", "Insolation", "Interpolations", "LazyArtifacts", "LazyBroadcast", "LinearAlgebra", "NCDatasets", "RootSolvers", "StaticArrays", "SurfaceFluxes", "Thermodynamics"]
+deps = ["ClimaComms", "ClimaCore", "ClimaDiagnostics", "ClimaParams", "ClimaTimeSteppers", "ClimaUtilities", "Dates", "DocStringExtensions", "Insolation", "Interpolations", "LazyArtifacts", "LazyBroadcast", "LinearAlgebra", "NCDatasets", "RootSolvers", "SciMLBase", "StaticArrays", "SurfaceFluxes", "Thermodynamics"]
 path = ".."
 uuid = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
-version = "1.7.0"
+version = "1.6.0"
 weakdeps = ["Adapt", "CairoMakie", "ClimaAnalysis", "DataFrames", "DelimitedFiles", "Downloads", "Flux", "GeoMakie", "InteractiveUtils", "JLD2", "Printf", "Statistics"]
 
     [deps.ClimaLand.extensions]
@@ -681,18 +681,6 @@ git-tree-sha1 = "a692f5e257d332de1e554e4566a4e5a8a72de2b2"
 uuid = "150eb455-5306-5404-9cee-2592286d6298"
 version = "0.6.4"
 
-[[deps.CoreMath]]
-deps = ["CoreMath_jll"]
-git-tree-sha1 = "8c0480f92b1b1796239156a1b9b1bfb1b39499b4"
-uuid = "b7a15901-be09-4a0e-87d2-2e66b0e09b5a"
-version = "0.1.0"
-
-[[deps.CoreMath_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "a692a4c1dc59a4b8bc0b6403876eb3250fde2bc3"
-uuid = "a38c48d9-6df1-5ac9-9223-b6ada3b5572b"
-version = "0.1.0+0"
-
 [[deps.Crayons]]
 git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
 uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -880,9 +868,9 @@ uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
 version = "1.0.7"
 
 [[deps.EnzymeCore]]
-git-tree-sha1 = "24bbb6fc8fb87eb71c1f8d00184a60fc22c63903"
+git-tree-sha1 = "990991b8aa76d17693a98e3a915ac7aa49f08d1a"
 uuid = "f151be2c-9106-41f4-ab19-57ee4f262869"
-version = "0.8.19"
+version = "0.8.18"
 weakdeps = ["Adapt", "ChainRulesCore"]
 
     [deps.EnzymeCore.extensions]
@@ -1126,17 +1114,10 @@ uuid = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e"
 version = "1.1.3"
 
 [[deps.FunctionWrappersWrappers]]
-deps = ["FunctionWrappers", "PrecompileTools", "TruncatedStacktraces"]
-git-tree-sha1 = "5201523536a43bf8aef3914b7f60b552b098ef8e"
+deps = ["FunctionWrappers"]
+git-tree-sha1 = "b104d487b34566608f8b4e1c39fb0b10aa279ff8"
 uuid = "77dc65aa-8811-40c2-897b-53d922fa7daf"
-version = "1.1.0"
-
-    [deps.FunctionWrappersWrappers.extensions]
-    FunctionWrappersWrappersEnzymeExt = ["Enzyme", "EnzymeCore"]
-
-    [deps.FunctionWrappersWrappers.weakdeps]
-    Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
-    EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
+version = "0.1.3"
 
 [[deps.Functors]]
 deps = ["Compat", "ConstructionBase", "LinearAlgebra", "Random"]
@@ -1462,10 +1443,10 @@ weakdeps = ["Unitful"]
     InterpolationsUnitfulExt = "Unitful"
 
 [[deps.IntervalArithmetic]]
-deps = ["CRlibm", "CoreMath", "MacroTools", "OpenBLASConsistentFPCSR_jll", "Printf", "Random", "RoundingEmulator"]
-git-tree-sha1 = "cf2ba7cef4e913abe44f5a9c7ade1c3776d7f222"
+deps = ["CRlibm", "MacroTools", "OpenBLASConsistentFPCSR_jll", "Printf", "Random", "RoundingEmulator"]
+git-tree-sha1 = "2cce1fed119ca7b6cc230c4a3b85202478af7924"
 uuid = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253"
-version = "1.0.4"
+version = "1.0.3"
 
     [deps.IntervalArithmetic.extensions]
     IntervalArithmeticArblibExt = "Arblib"
@@ -1552,9 +1533,9 @@ version = "1.7.1"
 
 [[deps.JSON]]
 deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"]
-git-tree-sha1 = "67c6f1f085cb2671c93fe34244c9cccde30f7a26"
+git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "1.5.0"
+version = "1.4.0"
 
     [deps.JSON.extensions]
     JSONArrowExt = ["ArrowTypes"]
@@ -1697,6 +1678,12 @@ weakdeps = ["Serialization"]
     [deps.LRUCache.extensions]
     SerializationExt = ["Serialization"]
 
+[[deps.LZO_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "1c602b1127f4751facb671441ca72715cc95938a"
+uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
+version = "2.10.3+0"
+
 [[deps.LaTeXStrings]]
 git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
 uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
@@ -2055,9 +2042,9 @@ weakdeps = ["Adapt", "CUDA"]
 
 [[deps.MutableArithmetics]]
 deps = ["LinearAlgebra", "SparseArrays", "Test"]
-git-tree-sha1 = "7c25249fc13a070f5ba433c50e21e22bb33c6fb0"
+git-tree-sha1 = "22df8573f8e7c593ac205455ca088989d0a2c7a0"
 uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0"
-version = "1.7.1"
+version = "1.6.7"
 
 [[deps.NCDatasets]]
 deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"]
@@ -2547,9 +2534,9 @@ version = "1.3.4"
 
 [[deps.RecursiveArrayTools]]
 deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "LinearAlgebra", "PrecompileTools", "RecipesBase", "StaticArraysCore", "SymbolicIndexingInterface"]
-git-tree-sha1 = "e4fd3369c78666a65ccec25dba28a0b181434ab2"
+git-tree-sha1 = "ea80ef4bc1e96be0a72a60c126527248eede8097"
 uuid = "731186ca-8d62-57ce-b412-fbd966d074cd"
-version = "3.52.0"
+version = "3.49.0"
 
     [deps.RecursiveArrayTools.extensions]
     RecursiveArrayToolsFastBroadcastExt = "FastBroadcast"
@@ -2668,9 +2655,9 @@ version = "0.1.0"
 
 [[deps.SciMLBase]]
 deps = ["ADTypes", "Accessors", "Adapt", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "Moshi", "PreallocationTools", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLLogging", "SciMLOperators", "SciMLPublic", "SciMLStructures", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface"]
-git-tree-sha1 = "908c0bf271604d09393a21c142116ab26f66f67c"
+git-tree-sha1 = "6e8c1bb72b768923d409edface0b3dcf61180d8d"
 uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
-version = "2.154.0"
+version = "2.152.1"
 
     [deps.SciMLBase.extensions]
     SciMLBaseChainRulesCoreExt = "ChainRulesCore"
@@ -3000,9 +2987,9 @@ version = "7.8.3+2"
 
 [[deps.SurfaceFluxes]]
 deps = ["RootSolvers", "Thermodynamics"]
-git-tree-sha1 = "f63a8a9eddd9d2efb229b421adda86437c8e230a"
+git-tree-sha1 = "830e95e01f1fc11ddbe4c4fcbabb017736c290aa"
 uuid = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
-version = "1.0.0"
+version = "0.15.1"
 weakdeps = ["ClimaParams"]
 
     [deps.SurfaceFluxes.extensions]
@@ -3141,12 +3128,6 @@ git-tree-sha1 = "4d4ed7f294cda19382ff7de4c137d24d16adc89b"
 uuid = "981d1d27-644d-49a2-9326-4793e63143c3"
 version = "0.1.0"
 
-[[deps.TruncatedStacktraces]]
-deps = ["InteractiveUtils", "MacroTools", "Preferences"]
-git-tree-sha1 = "ea3e54c2bdde39062abf5a9758a23735558705e1"
-uuid = "781d530d-4396-4725-bb49-402e4bee1e77"
-version = "1.4.0"
-
 [[deps.UUIDs]]
 deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

.gitignore

@@ -38,3 +38,16 @@ docs/site/
 # committed for packages, but should be committed for applications that require a static
 # environment.
 Manifest.toml
+
+# SLURM job logs
+*.out
+*.err
+
+# Julia distributed worker logs
+worker_*.log
+.julia-*.out
+
+# Generated calibration / UQ output (large JLD2 artifacts, not for version control)
+experiments/calibrate_dk_sor/output/
+experiments/callmip_uq_dk_sor/output_posterior_uq/
+experiments/callmip_uq_dk_sor/output_*/

experiments/calibrate_dk_sor/generate_observations.jl

@@ -0,0 +1,145 @@
+"""
+Pre-generate observation vector for DK-Sor calibration.
+
+Reads daily FluxNet observations (NEE, Qle, Qh) for 2004-2013, filters for
+days with valid data and daily mean wind speed < 5 m/s, and creates a single
+EKP.Observation. Saves to JLD2 for use by the calibration driver.
+
+Run once before calibration:
+    julia --project=.buildkite experiments/calibrate_dk_sor/generate_observations.jl
+"""
+
+using NCDatasets
+using Dates
+using Statistics
+using LinearAlgebra
+import JLD2
+import EnsembleKalmanProcesses as EKP
+import ClimaLand
+
+const FT = Float64
+const climaland_dir = abspath(joinpath(@__DIR__, "..", ".."))
+
+flux_nc_path = joinpath(
+    climaland_dir,
+    "DK_Sor",
+    "DK-Sor_daily_aggregated_1997-2013_FLUXNET2015_Flux.nc",
+)
+met_nc_path = joinpath(
+    climaland_dir,
+    "DK_Sor",
+    "DK-Sor_1997-2014_FLUXNET2015_Met.nc",
+)
+
+# Calibration years (after 1-year spinup starting 2003)
+cal_years = 2004:2013
+
+# ── Read flux observations ───────────────────────────────────────────────────
+flux_ds = NCDataset(flux_nc_path, "r")
+flux_times_dt = flux_ds["time"][:]
+
+nee_raw = Float64.(coalesce.(flux_ds["NEE_daily"][:], NaN))
+qle_raw = Float64.(coalesce.(flux_ds["Qle_daily"][:], NaN))
+qh_raw = Float64.(coalesce.(flux_ds["Qh_daily"][:], NaN))
+
+nee_uc_raw = Float64.(coalesce.(flux_ds["NEE_uc_daily"][:], NaN))
+qle_uc_raw = Float64.(coalesce.(flux_ds["Qle_uc_daily"][:], NaN))
+qh_uc_raw = Float64.(coalesce.(flux_ds["Qh_uc_daily"][:], NaN))
+close(flux_ds)
+
+flux_dates = Date.(flux_times_dt)
+
+# ── Compute daily mean wind speed from hourly Met data ────────────────────────
+met_ds = NCDataset(met_nc_path, "r")
+wind_data = Float64.(coalesce.(met_ds["Wind"][1, 1, :], NaN))
+met_times = met_ds["time"][:]
+close(met_ds)
+
+met_dates = Date.(met_times)
+
+# Average wind speed per day
+daily_wind = Dict{Date, Float64}()
+wind_by_day = Dict{Date, Vector{Float64}}()
+for (i, d) in enumerate(met_dates)
+    v = wind_data[i]
+    isnan(v) && continue
+    if !haskey(wind_by_day, d)
+        wind_by_day[d] = Float64[]
+    end
+    push!(wind_by_day[d], v)
+end
+for (d, vals) in wind_by_day
+    daily_wind[d] = mean(vals)
+end
+
+# ── Build valid-day mask ──────────────────────────────────────────────────────
+WIND_THRESHOLD = 5.0  # m/s
+
+cal_mask =
+    Date(first(cal_years), 1, 1) .<= flux_dates .<= Date(last(cal_years), 12, 31)
+
+valid_mask = cal_mask .&
+    .!isnan.(nee_raw) .& .!isnan.(qle_raw) .& .!isnan.(qh_raw) .&
+    (abs.(nee_raw) .< 1e10) .& (abs.(qle_raw) .< 1e10) .& (abs.(qh_raw) .< 1e10)
+
+# Apply wind filter
+for i in eachindex(valid_mask)
+    if valid_mask[i]
+        w = get(daily_wind, flux_dates[i], NaN)
+        if isnan(w) || w >= WIND_THRESHOLD
+            valid_mask[i] = false
+        end
+    end
+end
+
+obs_dates = flux_dates[valid_mask]
+nee_obs = nee_raw[valid_mask]
+qle_obs = qle_raw[valid_mask]
+qh_obs = qh_raw[valid_mask]
+nee_uc = nee_uc_raw[valid_mask]
+qle_uc = qle_uc_raw[valid_mask]
+qh_uc = qh_uc_raw[valid_mask]
+
+n_obs = length(obs_dates)
+println("Valid observation days: $n_obs ($(first(cal_years))-$(last(cal_years)), wind < $(WIND_THRESHOLD) m/s)")
+
+# ── Noise covariance ──────────────────────────────────────────────────────────
+nee_var = mean(filter(!isnan, nee_uc .^ 2))
+qle_var = mean(filter(!isnan, qle_uc .^ 2))
+qh_var = mean(filter(!isnan, qh_uc .^ 2))
+println(
+    "Noise variances - NEE: $(round(nee_var, sigdigits=3)) (gC/m²/d)², " *
+    "Qle: $(round(qle_var, sigdigits=3)) (W/m²)², " *
+    "Qh: $(round(qh_var, sigdigits=3)) (W/m²)²",
+)
+
+# ── Build single observation ─────────────────────────────────────────────────
+y_obs = vcat(nee_obs, qle_obs, qh_obs)
+noise_diag = vcat(
+    fill(nee_var, n_obs),
+    fill(qle_var, n_obs),
+    fill(qh_var, n_obs),
+)
+
+observation = EKP.Observation(
+    Dict(
+        "samples" => y_obs,
+        "covariances" => Diagonal(noise_diag),
+        "names" => "dk_sor_$(first(cal_years))_$(last(cal_years))",
+    ),
+)
+
+println("Observation vector length: $(length(y_obs)) (3 × $n_obs)")
+
+# ── Save ──────────────────────────────────────────────────────────────────────
+obs_filepath =
+    joinpath(climaland_dir, "experiments/calibrate_dk_sor/observations.jld2")
+JLD2.jldsave(
+    obs_filepath;
+    observation = observation,
+    obs_dates = obs_dates,
+    cal_years = collect(cal_years),
+    y_obs = y_obs,
+    noise_cov = Diagonal(noise_diag),
+)
+println("Saved to $obs_filepath")