Skip to content

Commit 9b023d0

Browse files
MarkusKonkbgruening
MarkusKonk
and
bgruening
authored
add new aquainfra tool: Copernicus CDS Downloader for AquaINFRA marine model (#200)
* add new tool: Copernicus CDS Downloader for AquaINFRA marine model * add new tool: Copernicus CDS Downloader for AquaINFRA marine model * add some linting * add some linting * add some linting * add some linting * add some linting * add citation * add creator * Update tools/ccds_download_era5_af/ccds_download_era5_af.xml Co-authored-by: Björn Grüning <bjoern@gruenings.eu> --------- Co-authored-by: Björn Grüning <bjoern@gruenings.eu>
1 parent 63ab3ae commit 9b023d0

4 files changed

Lines changed: 219 additions & 0 deletions

File tree

‎tools/ccds_download_era5_af/.shed.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
categories:
2+
- Ecology
3+
owner: ecology
4+
remote_repository_url: https://github.com/AquaINFRA/galaxy_vre
5+
homepage_url: https://github.com/AquaINFRA/galaxy_vre
6+
long_description: |
7+
Copernicus CDS Downloader for AquaINFRA marine model
8+
type: unrestricted
9+
auto_tool_repositories:
10+
name_template: "{{ tool_id }}"
11+
description_template: "Copernicus CDS Downloader for AquaINFRA marine model: {{ tool_name }}."

‎tools/ccds_download_era5_af/ccds_download_era5_af.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
MIN_LAT="${1}"
5+
MAX_LAT="${2}"
6+
MIN_LON="${3}"
7+
MAX_LON="${4}"
8+
START_DATE="${5}"
9+
END_DATE="${6}"
10+
11+
echo ">> Dowloading ERA5 atmospheric fields"
12+
13+
# Detect this script's directory (so we can find the Python helper file)
14+
TOOL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
15+
echo ">> startdate: ${START_DATE}"
16+
echo ">> Tool directory set to: ${TOOL_DIR}"
17+
18+
era5_xmin=`echo "${MIN_LON} - 0.5" | bc`
19+
era5_ymin=`echo "${MIN_LAT} - 0.5" | bc`
20+
era5_xmax=`echo "${MAX_LON} + 0.5" | bc`
21+
era5_ymax=`echo "${MAX_LAT} + 0.5" | bc`
22+
23+
echo ">> Requested bounding box is (lon_min, lon_max)=(${era5_xmin}, ${era5_xmax}) and (lat_min, lat_max)=(${era5_ymin}, ${era5_ymax})"
24+
25+
max_attempts=50
26+
attempt=1
27+
28+
#Download the data
29+
30+
while (( attempt <= max_attempts )); do
31+
python3 ${TOOL_DIR}/get_era5.py ${START_DATE} ${END_DATE} \
32+
${era5_xmin} ${era5_xmax} ${era5_ymin} ${era5_ymax}
33+
wait
34+
if [[ -f "ccds_era5_af_data.zip" ]]; then
35+
echo ">> File is present after $attempt attempts"
36+
break
37+
else
38+
if (( attempt == max_attempts )); then
39+
echo ">> File not found after $max_attempts attempts"
40+
exit 1
41+
else
42+
echo ">> Attempt $((attempt))/$max_attempts: File still missing"
43+
((attempt++))
44+
fi
45+
fi
46+
done

‎tools/ccds_download_era5_af/ccds_download_era5_af.xml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
<tool id="ccds_download_era5_af" name="Copernicus CDS Downloader for AquaINFRA marine model" version="1.0.0" profile="22.05">
2+
<description>
3+
Download ERA5 data from the Copernicus Climate Data Store to provide as input to the AquaINFRA marine model. Products: ERA5 hourly data on single levels from 1940 to present, Reanalysis.
4+
</description>
5+
<creator>
6+
<organization name="EOSC AquaINFRA" url="https://aquainfra.eu/"/>
7+
</creator>
8+
<requirements>
9+
<requirement type="package" version="1.41.0">boto3</requirement>
10+
<requirement type="package" version="1.41.0">botocore</requirement>
11+
<requirement type="package" version="5.2.37">bash</requirement>
12+
<requirement type="package" version="3.14.0">python</requirement>
13+
<requirement type="package" version="2.3.3">pandas</requirement>
14+
<requirement type="package" version="0.7.7">cdsapi</requirement>
15+
<!--<credentials name="cds_api" version="1.0">
16+
<variable name="token" inject_as_env="CDS_TOKEN" />
17+
</credentials>-->
18+
</requirements>
19+
<command detect_errors="exit_code"><![CDATA[
20+
bash '$cmems_credentials' &&
21+
22+
bash '$__tool_directory__/ccds_download_era5_af.sh' \
23+
"$min_lat" "$max_lat" \
24+
"$min_lon" "$max_lon" \
25+
"$start_date" "$end_date"
26+
]]></command>
27+
<configfiles>
28+
<configfile name="cmems_credentials">
29+
#import os
30+
#set $cmems_username = $__user__.extra_preferences.get('cmems_username', "")
31+
#set $cmems_password = $__user__.extra_preferences.get('cmems_password', "")
32+
#if $cmems_username == "" or $cmems_password == ""
33+
#set $cmems_username = os.getenv('CMEMS_USERNAME', '')
34+
#set $cmems_password = os.getenv('CMEMS_PASSWORD', '')
35+
#end if
36+
export CMEMS_USERNAME='$cmems_username'
37+
export CMEMS_PASSWORD='$cmems_password'
38+
</configfile>
39+
</configfiles>
40+
<inputs>
41+
<param name="min_lat" type="float" label="Min latitude (ºN)" min="-180" max="180" optional="false" help="Minimum latitude value for the data subset (southern boundary latitude)." />
42+
<param name="max_lat" type="float" label="Max latitude (ºN)" min="-180" max="180" optional="false" help="Maximum latitude value for the data subset (northern boundary latitude)." />
43+
<param name="min_lon" type="float" label="Min longitude (ºE)" min="-180" max="180" optional="false" help="Minimum longitude value for the data subset (western boundary longitude)." />
44+
<param name="max_lon" type="float" label="Max longitude (ºE)" min="-180" max="180" optional="false" help="Maximum longitude value for the data subset (eastern boundary longitude)." />
45+
<param name="start_date" type="text" label="Start date (YYYY-MM-DD)" optional="false" help="The start datetime of the temporal data subset.">
46+
<validator type="regex" message="Date must be in YYYY-MM-DD format">^\d{4}-\d{2}-\d{2}$</validator>
47+
</param>
48+
<param name="end_date" type="text" label="End date (YYYY-MM-DD)" optional="false" help="The end datetime of the temporal data subset.">
49+
<validator type="regex" message="Date must be in YYYY-MM-DD format">^\d{4}-\d{2}-\d{2}$</validator>
50+
</param>
51+
</inputs>
52+
<outputs>
53+
<data name="zip_output" format="zip" from_work_dir="ccds_era5_af_data.zip" label="ccds_era5_af_data.zip"/>
54+
</outputs>
55+
<tests>
56+
<test expect_failure="true">
57+
<param name="min_lat" value="38.42"/>
58+
<param name="max_lat" value="43.60"/>
59+
<param name="min_lon" value="-0.46"/>
60+
<param name="max_lon" value="6.1728"/>
61+
<param name="start_date" value="2022-01-01"/>
62+
<param name="end_date" value="2022-01-02"/>
63+
<assert_stdout>
64+
<has_text text=">> startdate: 2022-01-01"/>
65+
</assert_stdout>
66+
</test>
67+
</tests>
68+
<help format="markdown"><![CDATA[
69+
This tool downloads ERA5 data subsets from the **Copernicus Climate Data Store (ECMWF)** using the **CDSAPI** to provide later as input to the AquaINFRA marine model.
70+
71+
### Requirements
72+
73+
To use this tool, users must:
74+
75+
- Have a valid account on the Copernicus Climate Data Store: https://cds.climate.copernicus.eu
76+
- Have accepted the *Terms of Use* for the **ERA5 hourly data on single levels** dataset. This must be done manually on the dataset’s webpage (at the bottom of the download form).
77+
78+
### Outputs
79+
80+
The tool produces a ZIP file containing two NetCDF (`.nc`) files:
81+
82+
- `data_stream-oper_stepType-accum.nc` — accumulated variables
83+
- `data_stream-oper_stepType-instant.nc` — instantaneous variables
84+
85+
These files can be used as inputs for the AquaINFRA marine model tool..
86+
]]></help>
87+
<citations>
88+
<citation type="bibtex">
89+
@misc{netcdf4,
90+
title={netcdf4},
91+
author={Jeff Whitaker},
92+
url={https://pypi.org/project/netCDF4/}
93+
}
94+
</citation>
95+
</citations>
96+
</tool>

‎tools/ccds_download_era5_af/get_era5.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import os
2+
import sys
3+
from datetime import datetime
4+
5+
import cdsapi
6+
7+
import pandas as pd
8+
9+
# Read arguments
10+
# get-era5.py INITIAL_DATE FINAL_DATE XMIN XMAX YMIN YMAX
11+
12+
SDATE = sys.argv[1]
13+
EDATE = sys.argv[2]
14+
WEST = sys.argv[3]
15+
EAST = sys.argv[4]
16+
SOUTH = sys.argv[5]
17+
NORTH = sys.argv[6]
18+
19+
start_date = datetime.strptime(SDATE, "%Y-%m-%d")
20+
end_date = datetime.strptime(EDATE, "%Y-%m-%d")
21+
22+
date_list = list(
23+
pd.date_range(start_date, end_date, freq='D')
24+
.strftime('%Y-%m-%d')
25+
)
26+
27+
VARIABLES = [
28+
'10m_u_component_of_wind',
29+
'10m_v_component_of_wind',
30+
'2m_dewpoint_temperature',
31+
'2m_temperature',
32+
'mean_sea_level_pressure',
33+
'surface_solar_radiation_downwards',
34+
'surface_thermal_radiation_downwards',
35+
'total_precipitation',
36+
]
37+
38+
OUTPUT_FILENAME = 'ccds_era5_af_data.zip'
39+
40+
c = cdsapi.Client(
41+
url="https://cds.climate.copernicus.eu/api",
42+
key=os.environ["CDS_TOKEN"]
43+
)
44+
45+
fl = c.retrieve(
46+
'reanalysis-era5-single-levels',
47+
{
48+
'product_type': 'reanalysis',
49+
'format': 'netcdf',
50+
'variable': VARIABLES,
51+
'date': date_list,
52+
'time': [
53+
'00:00', '01:00', '02:00',
54+
'03:00', '04:00', '05:00',
55+
'06:00', '07:00', '08:00',
56+
'09:00', '10:00', '11:00',
57+
'12:00', '13:00', '14:00',
58+
'15:00', '16:00', '17:00',
59+
'18:00', '19:00', '20:00',
60+
'21:00', '22:00', '23:00',
61+
],
62+
'area': [NORTH, WEST, SOUTH, EAST],
63+
'data_format': "netcdf4",
64+
"download_format": "zip"
65+
},
66+
OUTPUT_FILENAME)

0 commit comments

Comments
 (0)