Skip to content

Commit 03b4217

Browse files
axel-lauervaleriupredoirbeucher
authored
New version of JRA-25 CMORizer (#3470)
Co-authored-by: Valeriu Predoi <valeriu.predoi@gmail.com> Co-authored-by: Romain Beucher <romain.beucher@anu.edu.au>
1 parent d968227 commit 03b4217

6 files changed

Lines changed: 219 additions & 82 deletions

File tree

‎doc/sphinx/source/input.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol
393393
+----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
394394
| JMA-TRANSCOM | nbp (Lmon), fgco2 (Omon) | 3 | Python |
395395
+----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
396-
| JRA-25 | clt, hus, prw, rlut, rlutcs, rsut, rsutcs (Amon) | 2 | Python |
396+
| JRA-25 | cl, clt, clwvi, hus, prw, rlut, rlutcs, rsut, rsutcs, ta, tas, tcw, wap (Amon) | 2 | Python |
397397
+----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
398398
| JRA-55 | cli, clivi, clw, clwvi, clt, prw, rlus, rlut, rlutcs, rsus, rsuscs, rsut, rsutcs, ta, tas, wap (Amon)| 2 | Python |
399399
+----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+

‎esmvaltool/cmorizers/data/cmor_config/JRA-25.yml‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,45 +6,75 @@ attributes:
66
tier: 2
77
modeling_realm: reanaly
88
project_id: OBS6
9-
source: 'https://esgf.nccs.nasa.gov/thredds/fileServer/CREATE-IP/reanalysis/JMA/JRA-25/JRA-25/'
9+
source: 'https://data.rda.ucar.edu/ds625.1'
1010
reference: 'jra_25'
1111
comment: |
1212
''
1313
1414
# Variables to cmorize
1515
variables:
16-
clt_month:
16+
cl:
17+
short_name: cl
18+
mip: Amon
19+
raw: TCDC_HBL
20+
file: 'fcst_phy3m.{year}*.nc'
21+
clt:
1722
short_name: clt
1823
mip: Amon
19-
raw: clt
20-
file: clt_Amon_reanalysis_JRA-25_197901-201312.nc
21-
hus_month:
24+
raw: TCDC_COL
25+
file: 'fcst_phy2m.{year}*.nc'
26+
clwvi:
27+
short_name: clwvi
28+
mip: Amon
29+
raw: CWAT_COL
30+
file: 'fcst_phy2m.{year}*.nc'
31+
hus:
2232
short_name: hus
2333
mip: Amon
24-
raw: hus
25-
file: hus_Amon_reanalysis_JRA-25_197901-201312.nc
26-
prw_month:
34+
raw: SPFH_PRS
35+
file: 'anl_p.{year}*.nc'
36+
prw:
2737
short_name: prw
2838
mip: Amon
29-
raw: prw
30-
file: prw_Amon_reanalysis_JRA-25_197901-201312.nc
31-
rlut_month:
39+
raw: PWAT_COL
40+
file: 'fcst_phy2m.{year}*.nc'
41+
rlut:
3242
short_name: rlut
3343
mip: Amon
34-
raw: rlut
35-
file: rlut_Amon_reanalysis_JRA-25_197901-201312.nc
36-
rlutcs_month:
44+
raw: ULWRF_TOA
45+
file: 'fcst_phy2m.{year}*.nc'
46+
rlutcs:
3747
short_name: rlutcs
3848
mip: Amon
39-
raw: rlutcs
40-
file: rlutcs_Amon_reanalysis_JRA-25_197901-201312.nc
41-
rsut_month:
49+
raw: CSULF_TOA
50+
file: 'fcst_phy2m.{year}*.nc'
51+
rsut:
4252
short_name: rsut
4353
mip: Amon
44-
raw: rsut
45-
file: rsut_Amon_reanalysis_JRA-25_197901-201312.nc
46-
rsutcs_month:
54+
raw: USWRF_TOA
55+
file: 'fcst_phy2m.{year}*.nc'
56+
rsutcs:
4757
short_name: rsutcs
4858
mip: Amon
49-
raw: rsutcs
50-
file: rsutcs_Amon_reanalysis_JRA-25_197901-201312.nc
59+
raw: CSUSF_TOA
60+
file: 'fcst_phy2m.{year}*.nc'
61+
ta:
62+
short_name: ta
63+
mip: Amon
64+
raw: TMP_PRS
65+
file: 'anl_p.{year}*.nc'
66+
tas:
67+
short_name: tas
68+
mip: Amon
69+
raw: TMP_SFC
70+
file: 'anl_p.{year}*.nc'
71+
tcw:
72+
short_name: tcw
73+
mip: Amon
74+
raw: CWAT_PRS
75+
file: 'anl_p.{year}*.nc'
76+
wap:
77+
short_name: wap
78+
mip: Amon
79+
raw: VVEL_PRS
80+
file: 'anl_chipsi.{year}*.nc'

‎esmvaltool/cmorizers/data/datasets.yml‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -894,17 +894,13 @@ datasets:
894894
895895
JRA-25:
896896
tier: 2
897-
source: https://esgf.nccs.nasa.gov/thredds/fileServer/CREATE-IP/reanalysis/JMA/JRA-25/JRA-25/
898-
last_access: 2022-11-17
897+
source: https://data.rda.ucar.edu/ds625.1
898+
last_access: 2023-06-14
899899
info: |
900900
Download the following files:
901-
mon/atmos/clt/clt_Amon_reanalysis_JRA-25_197901-201312.nc
902-
mon/atmos/hus/hus_Amon_reanalysis_JRA-25_197901-201312.nc
903-
mon/atmos/prw/prw_Amon_reanalysis_JRA-25_197901-201312.nc
904-
mon/atmos/rlut/rlut_Amon_reanalysis_JRA-25_197901-201312.nc
905-
mon/atmos/rlutcs/rlutcs_Amon_reanalysis_JRA-25_197901-201312.nc
906-
mon/atmos/rsut/rsut_Amon_reanalysis_JRA-25_197901-201312.nc
907-
mon/atmos/rsutcs/rsutcs_Amon_reanalysis_JRA-25_197901-201312.nc
901+
fcst_phy2m/fcst_phy2m.yyyymm.nc
902+
anl_chipsi/fcst_phy2m.yyyymm.nc
903+
anl_p/anl_p.yyyymm.nc
908904
909905
JRA-55:
910906
tier: 2

‎esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
"""Script to download JRA-25 from ESGF."""
1+
"""Script to download JRA-25 from RDA."""
22

33
import logging
44
import os
5+
from datetime import datetime
6+
7+
from dateutil import relativedelta
58

69
from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
710

@@ -37,36 +40,32 @@ def download_dataset(
3740

3841
os.makedirs(downloader.local_folder, exist_ok=True)
3942

40-
url = (
41-
"https://esgf.nccs.nasa.gov/thredds/fileServer/CREATE-IP/"
42-
"reanalysis/JMA/JRA-25/JRA-25/mon/atmos/"
43-
)
43+
if start_date is None:
44+
start_date = datetime(1979, 1, 1)
45+
if end_date is None:
46+
end_date = datetime(2007, 12, 31)
47+
loop_date = start_date
4448

45-
downloader.download_file(
46-
url + "clt/clt_Amon_reanalysis_JRA-25_197901-201312.nc",
47-
wget_options=[],
48-
)
49-
downloader.download_file(
50-
url + "hus/hus_Amon_reanalysis_JRA-25_197901-201312.nc",
51-
wget_options=[],
52-
)
53-
downloader.download_file(
54-
url + "prw/prw_Amon_reanalysis_JRA-25_197901-201312.nc",
55-
wget_options=[],
56-
)
57-
downloader.download_file(
58-
url + "rlut/rlut_Amon_reanalysis_JRA-25_197901-201312.nc",
59-
wget_options=[],
60-
)
61-
downloader.download_file(
62-
url + "rlutcs/rlutcs_Amon_reanalysis_JRA-25_197901-201312.nc",
63-
wget_options=[],
64-
)
65-
downloader.download_file(
66-
url + "rsut/rsut_Amon_reanalysis_JRA-25_197901-201312.nc",
67-
wget_options=[],
68-
)
69-
downloader.download_file(
70-
url + "rsutcs/rsutcs_Amon_reanalysis_JRA-25_197901-201312.nc",
71-
wget_options=[],
72-
)
49+
# download files
50+
51+
url = "https://data.rda.ucar.edu/ds625.1"
52+
download_options = ["--no-check-certificate"]
53+
54+
# define files to download
55+
56+
files = ["fcst_phy2m", "fcst_phy3m", "anl_chipsi", "anl_p"]
57+
58+
# download data
59+
60+
while loop_date <= end_date:
61+
year = loop_date.year
62+
month = f"{loop_date.month:0>2}"
63+
64+
for basename in files:
65+
fname = f"{basename}.{year}{month}.nc"
66+
# download file
67+
downloader.download_file(
68+
url + "/" + basename + "/" + fname, download_options
69+
)
70+
71+
loop_date += relativedelta.relativedelta(months=1)

‎esmvaltool/cmorizers/data/formatters/datasets/jra_25.py‎

Lines changed: 123 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,43 +5,132 @@
55
Tier 2: other freely-available dataset.
66
77
Source
8-
ESGF:
9-
https://esgf.nccs.nasa.gov/thredds/fileServer/CREATE-IP/
10-
reanalysis/JMA/JRA-25/JRA-25/mon/atmos/
11-
8+
https://data.rda.ucar.edu/ds625.1
129
Last access
13-
20221122
10+
2023-06-14
1411
1512
Download and processing instructions
1613
see download script cmorizers/data/downloaders/datasets/jra_25.py
1714
"""
1815

1916
import copy
17+
import glob
2018
import logging
2119
import os
20+
from datetime import datetime
2221

2322
import iris
23+
from dateutil import relativedelta
24+
from esmvalcore.preprocessor import extract_levels
2425
from iris import NameConstraint
26+
from iris.util import equalise_attributes
2527

2628
from esmvaltool.cmorizers.data import utilities as utils
2729

2830
logger = logging.getLogger(__name__)
2931

3032

31-
def _extract_variable(short_name, var, filename, cfg, in_dir, out_dir):
33+
def _extract_variable(short_name, var, in_files, cfg, out_dir):
3234
"""Extract variable."""
3335
# load data
34-
filepath = os.path.join(in_dir, filename)
3536
raw_var = var.get("raw", short_name)
36-
cube = iris.load_cube(filepath, NameConstraint(var_name=raw_var))
37+
rawcubes = iris.load(in_files, NameConstraint(var_name=raw_var))
38+
39+
equalise_attributes(rawcubes)
40+
41+
# check if data are on hybrid levels
42+
43+
hybrid_levels = False
44+
45+
for coord in rawcubes[0].coords():
46+
if (
47+
coord.standard_name
48+
== "atmosphere_hybrid_sigma_pressure_coordinate"
49+
):
50+
hybrid_levels = True
51+
break
52+
53+
# If (3-dim) data are on hybrid levels then regrid data to pressure levels.
54+
# The pressure levels are taken from the JRA-25 files that provide JRA-25
55+
# data on pressure levels (e.g. anl_p.yyyymm.nc).
56+
57+
if hybrid_levels:
58+
pcubes = iris.cube.CubeList([])
59+
60+
for cube in rawcubes:
61+
pcube = extract_levels(
62+
cube,
63+
[
64+
100000,
65+
92500,
66+
85000,
67+
70000,
68+
60000,
69+
50000,
70+
40000,
71+
30000,
72+
25000,
73+
20000,
74+
15000,
75+
10000,
76+
7000,
77+
5000,
78+
3000,
79+
2000,
80+
1000,
81+
700,
82+
500,
83+
300,
84+
200,
85+
100,
86+
40,
87+
],
88+
"linear",
89+
coordinate="air_pressure",
90+
)
91+
# remove auxiliary coordinate 'Surface_pressure'
92+
pcube.remove_coord("Surface_pressure")
93+
# rename dimension air_pressure to plev
94+
coord = pcube.coord("air_pressure")
95+
coord.rename("plev")
96+
coord.var_name = "plev"
97+
98+
pcubes.append(pcube)
99+
100+
cube = pcubes.concatenate_cube()
101+
else:
102+
cube = rawcubes.concatenate_cube()
37103

38-
# Fix metadata
39104
cmor_info = cfg["cmor_table"].get_variable(var["mip"], short_name)
105+
106+
try:
107+
cube.convert_units(cmor_info.units)
108+
except Exception as ex: # pylint: disable=broad-except
109+
logger.warning(
110+
"Warning: could not convert units from %s to %s (%r)",
111+
cube.units,
112+
cmor_info.units,
113+
ex,
114+
)
115+
116+
# Fix coordinates
117+
utils.fix_coords(cube)
118+
for coord in cube.coords():
119+
if coord.var_name == "plev":
120+
coord.standard_name = "air_pressure"
121+
coord.long_name = "pressure"
122+
coord.attributes["positive"] = "up"
123+
124+
# Fix metadata
40125
attrs = copy.deepcopy(cfg["attributes"])
41126
attrs["mip"] = var["mip"]
42127
utils.fix_var_metadata(cube, cmor_info)
43128
utils.set_global_atts(cube, attrs)
44129

130+
# Add height2m if needed
131+
if "height2m" in cmor_info.dimensions:
132+
utils.add_height2m(cube)
133+
45134
# Save variable
46135
utils.save_variable(
47136
cube, short_name, out_dir, attrs, unlimited_dimensions=["time"]
@@ -50,11 +139,28 @@ def _extract_variable(short_name, var, filename, cfg, in_dir, out_dir):
50139

51140
def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
52141
"""Cmorization func call."""
53-
# Run the cmorization
54-
for short_name, var in cfg["variables"].items():
55-
short_name = var["short_name"]
56-
filename = var["file"]
57-
logger.info(
58-
"CMORizing variable '%s' from file '%s'", short_name, filename
59-
)
60-
_extract_variable(short_name, var, filename, cfg, in_dir, out_dir)
142+
if start_date is None:
143+
start_date = datetime(1979, 1, 1)
144+
if end_date is None:
145+
end_date = datetime(2007, 12, 31)
146+
loop_date = start_date
147+
148+
while loop_date <= end_date:
149+
year = loop_date.year
150+
151+
for short_name, var in cfg["variables"].items():
152+
if "short_name" not in var:
153+
var["short_name"] = short_name
154+
155+
# Now get list of files
156+
filepattern = os.path.join(
157+
in_dir + "/" + var["file"].format(year=year)
158+
)
159+
print("*** " + filepattern)
160+
in_files = glob.glob(filepattern)
161+
if not in_files:
162+
logger.warning("Warning: no data found for %d", year)
163+
continue
164+
_extract_variable(short_name, var, in_files, cfg, out_dir)
165+
166+
loop_date += relativedelta.relativedelta(years=1)

0 commit comments

Comments
 (0)