Update calipso_cloudsat artifact with monthly data

Author: ph-kev

calipso_cloudsat/Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.11.5"
+julia_version = "1.11.8"
 manifest_format = "2.0"
-project_hash = "ba09a07cf72bb92a3c83575effee91f0237dfb76"
+project_hash = "7aa728f622ede69eaf4804ee6b6053fc32fb5113"
 
 [[deps.ArgCheck]]
 git-tree-sha1 = "f9e9a66c9b7be1ad7372bbd9b062d9230c30c5ce"

calipso_cloudsat/OutputArtifacts.toml

@@ -1,8 +1,4 @@
 [calipso_cloudsat]
 git-tree-sha1 = "dae77bc4ff79194ee6e794c1181bec1bebd67ea9"
 [calipso_cloudsat_lowres]
-git-tree-sha1 = "a76ee4474873a2cf366fb0755f7fbac647987082"
-
-    [[calipso_cloudsat_lowres.download]]
-    sha256 = "deb774f613c4c3668cb1b7b272711dcef699c318ff03f5b8f3ea83d5a3d50926"
-    url = "https://caltech.box.com/shared/static/ywx8ssv70d7b7t6bdp148su955tbrkoq.gz"
+git-tree-sha1 = "c1c0a16127bd547124cdf7288140002d74846efe"

calipso_cloudsat/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 ClimaArtifactsHelper = "6ffa2572-8378-4377-82eb-ea11db28b255"
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c"

calipso_cloudsat/README.md

@@ -2,8 +2,11 @@
 
 This artifact repackages data coming from the [3S-GEOPROF-COMB
 Dataset](https://zenodo.org/records/12768877) and contains seasonal data from
-2006 JJA (June, July, August) to 2020 JJA. In particular, the data is from
-`radarlidar_seasonal_2.5x2.5.zip` and `radarlidar_seasonal_10x10.zip`.
+2006 JJA (June, July, August) to 2020 JJA and monthly data from June 2006 to
+August 2020. In particular, the seasonal data is from
+`radarlidar_seasonal_2.5x2.5.zip` and `radarlidar_seasonal_10x10.zip` and the
+monthly data is from `radarlidar_monthly_2.5x2.5.zip` and
+`radarlidar_monthly_10x10.zip`.
 
 ## Data
 
@@ -54,7 +57,7 @@ For more information about the data, refer to the documentation on
 
 1. Julia
 2. Python
-3. 3.9GiB to download and extract the data from the downloaded zip files.
+3. 16GiB to download and extract the data from the downloaded zip files.
 
 ## Usage
 
@@ -74,11 +77,16 @@ while any global attribute that remains the same is included as a single value.
 
 ## Files
 
-The files included are `radarlidar_seasonal_2.5x2.5.nc` and
+The files included for seasonal data are `radarlidar_seasonal_2.5x2.5.nc` and
 `radarlidar_seasonal_10x10.nc`, a lower resolution version of
 `radarlidar_seasonal_2.5x2.5.nc`. The size of `radarlidar_seasonal_2.5x2.5.nc`
 is 3.1GiB and the size of `radarlidar_seasonal_10x10.nc` is 196MiB.
 
+The files included for monthly data are `radarlidar_monthly_2.5x2.5.nc` and
+`radarlidar_monthly_10x10.nc`, a lower resolution version of
+`radarlidar_monthly_2.5x2.5.nc`. The size of `radarlidar_monthly_2.5x2.5.nc`
+is 9.2GiB and the size of `radarlidar_monthly_10x10.nc` is 588MiB.
+
 ## License
 
 Creative Commons Attribution 4.0 International.

calipso_cloudsat/create_artifact.jl

@@ -1,6 +1,7 @@
 using NCDatasets
 using DataStructures
 using ClimaArtifactsHelper
+import Dates
 
 """
     parse_year_season(path)
@@ -12,12 +13,25 @@ The seasons MAM, JJA, SON, and DJF map to 1, 2, 3, and 4.
 function parse_year_season(path)
     # File name should look like "2008-MAM-..."
     filename = last(splitpath(path))
-    s = first(filename, 8)
+    s = split(filename, "_")[1]
     year, season = split(s, "-")
     SEASON_ORDER = Dict("MAM" => 1, "JJA" => 2, "SON" => 3, "DJF" => 4)
     return (parse(Int, year), SEASON_ORDER[season])
 end
 
+"""
+    parse_year_month(path)
+
+Given a file `path`, return the year and month as integers.
+"""
+function parse_year_month(path)
+    # File name should look like "2008-MAM-..."
+    filename = last(splitpath(path))
+    s = split(filename, "_")[1]
+    year, month = split(s, "-")
+    return parse.(Int, (year, month))
+end
+
 """
     get_and_sort_files(dir)
 
@@ -31,7 +45,13 @@ function get_and_sort_files(dir)
     # For each year, the order of the seasons are "MAM", "JJA", "SON", "DJF"
     # Note that the first month is used, so 2006-DJF refers to Dec 2006, Jan
     # 2007, and Feb 2007
-    sort!(files, by = parse_year_season)
+    if occursin("seasonal", dir)
+        sort!(files, by = parse_year_season)
+    elseif occursin("monthly", dir)
+        sort!(files, by = parse_year_month)
+    else
+        error("Only support files with monthly or seasonal periods")
+    end
     return files
 end
 
@@ -81,6 +101,16 @@ function stitch_cloud_data(files, output_filepath)
         defVar(ds, varname, data, dimnames(mfds[varname]), attrib = mfds[varname].attrib)
     end
 
+    # Seasonal files have a time dimension, but monthly files doesn't
+    if occursin("monthly", first(files))
+        @info "Adding time dimension for monthly dataset"
+        year_and_month = parse_year_month.(files)
+        data = [Dates.DateTime(year, month) for (year, month) in year_and_month]
+        attrib = Dict("units" => "days since 2006-06-01 00:00:00", "calendar" => "proleptic_gregorian")
+        first(data) == Dates.DateTime(2006, 6) || error("The first date is not 2006-06-01")
+        defVar(ds, "time", data, ("time", ), attrib = attrib)
+    end
+
     # Close datasets
     close(ds)
     close(mfds)
@@ -179,7 +209,7 @@ function make_artifact(filepath_names)
     end
 
     for filepath_name in filepath_names
-        if endswith(filepath_name, "radarlidar_seasonal_2.5x2.5.nc")
+        if endswith(filepath_name, "_2.5x2.5.nc")
             filename = last(splitpath(filepath_name))
             mv(filepath_name, joinpath(output_dir, filename), force = true)
         end
@@ -196,7 +226,7 @@ function make_artifact(filepath_names)
     end
 
     for filepath_name in filepath_names
-        if endswith(filepath_name, "radarlidar_seasonal_10x10.nc")
+        if endswith(filepath_name, "_10x10.nc")
             filename = last(splitpath(filepath_name))
             mv(filepath_name, joinpath(output_dir_lowres, filename), force = true)
         end
@@ -208,13 +238,15 @@ function make_artifact(filepath_names)
     )
 end
 
-radarlidar_dir_2_5 =
-    joinpath(dirname(@__FILE__), "radarlidar_seasonal_data", "radarlidar_seasonal_2.5x2.5")
-radarlidar_dir_10 =
-    joinpath(dirname(@__FILE__), "radarlidar_seasonal_data", "radarlidar_seasonal_10x10")
+radarlidar_dirs = [
+    joinpath(dirname(@__FILE__), "radarlidar_data", "radarlidar_seasonal_2.5x2.5")
+    joinpath(dirname(@__FILE__), "radarlidar_data", "radarlidar_seasonal_10x10")
+    joinpath(dirname(@__FILE__), "radarlidar_data", "radarlidar_monthly_2.5x2.5")
+    joinpath(dirname(@__FILE__), "radarlidar_data", "radarlidar_monthly_10x10")
+]
 
 # Check if the directory exists
-for directory in (radarlidar_dir_10, radarlidar_dir_2_5)
+for directory in radarlidar_dirs
     if !isdir(directory)
         error("$directory is not a directory. Check the file path passed in")
     end
@@ -223,7 +255,7 @@ end
 # Loop over the directories containing the NetCDF files and create a netcdf file
 # from them
 filepath_names = []
-for directory in (radarlidar_dir_10, radarlidar_dir_2_5)
+for directory in radarlidar_dirs
     files = get_and_sort_files(directory)
     filename = last(splitpath(directory)) * ".nc"
     filepath_name = joinpath(dirname(directory), filename)

calipso_cloudsat/download_cloudsat.py

@@ -13,20 +13,24 @@ def download_and_unzip_cloudsat():
     # Download data
     seasonal_10_zip = dir_path.joinpath("radarlidar_seasonal_10x10.zip")
     seasonal_2_5_zip = dir_path.joinpath("radarlidar_seasonal_2.5x2.5.zip")
-    zip_files = [seasonal_10_zip, seasonal_2_5_zip]
+    monthly_10_zip = dir_path.joinpath("radarlidar_monthly_10x10.zip")
+    monthly_2_5_zip = dir_path.joinpath("radarlidar_monthly_2.5x2.5.zip")
+    zip_files = [seasonal_10_zip, seasonal_2_5_zip, monthly_10_zip, monthly_2_5_zip]
     download_links = [
         "https://zenodo.org/records/12768877/files/radarlidar_seasonal_10x10.zip?download=1",
         "https://zenodo.org/records/12768877/files/radarlidar_seasonal_2.5x2.5.zip?download=1",
+        "https://zenodo.org/records/12768877/files/radarlidar_monthly_10x10.zip?download=1",
+        "https://zenodo.org/records/12768877/files/radarlidar_monthly_2.5x2.5.zip?download=1",
     ]
     for download_link, zip_file in zip(download_links, zip_files):
         with urlopen(download_link) as in_stream, open(zip_file, "wb") as out_file:
             copyfileobj(in_stream, out_file)
 
     # Find files and unzip them
-    unzip_directory = dir_path.joinpath("radarlidar_seasonal_data")
+    unzip_directory = dir_path.joinpath("radarlidar_data")
     if not os.path.isdir(unzip_directory):
         os.mkdir(unzip_directory)
-    for zip_file in [seasonal_10_zip, seasonal_2_5_zip]:
+    for zip_file in zip_files:
         with zipfile.ZipFile(str(zip_file), "r") as zip_ref:
             zip_ref.extractall(unzip_directory)
     return None
@@ -35,5 +39,5 @@ def download_and_unzip_cloudsat():
 if __name__ == "__main__":
     download_and_unzip_cloudsat()
     print(
-        "The data should be downloaded and stored in radarlidar_seasonal_data. You can now delete the zip files"
+        "The data should be downloaded and stored in radarlidar_data. You can now delete the zip files"
     )