1- """Script to download ESACCI-AEROSOL from CCI CEDA ftp."""
1+ """Script to download ESACCI-AEROSOL data.
22
3- from datetime import datetime
3+ From CCI CEDA ftp (AATSR) and via Copernicus Climate Data Store (SLSTR).
44
5+ Note: For downloading data from the Copernicus Climate Data Store (CDS), an
6+ account is needed. A CDS account can be created at
7+ https://cds.climate.copernicus.eu/
8+ After creating an account and logging in, the following two steps are
9+ required:
10+ (1) Setup a CDS Api key. Instructions can be found here:
11+ https://cds.climate.copernicus.eu/how-to-api
12+ (2) Agree to the "Licence to use Copernicus Products":
13+ After logging in, click on your name in the top right corner,
14+ select "Your profile", click on the "Licenses" tab and check
15+ "Licence to use Copernicus Products".
16+ """
17+
18+ import datetime
19+ import gzip
20+ import logging
21+ import shutil
22+ import zipfile
23+
24+ import cdsapi
525from dateutil import relativedelta
626
727from esmvaltool .cmorizers .data .downloaders .ftp import CCIDownloader
828
29+ logger = logging .getLogger (__name__ )
30+
931
1032def download_dataset (
1133 original_data_dir ,
@@ -33,9 +55,27 @@ def download_dataset(
3355 Overwrite already downloaded files
3456 """
3557 if start_date is None :
36- start_date = datetime (1997 , 1 , 1 )
58+ aatsr_year1 = 1997
59+ slstr_year1 = 2017
60+ else :
61+ aatsr_year1 = start_date .year
62+ slstr_year1 = start_date .year
63+ if end_date is None :
64+ aatsr_year2 = 2011
65+ slstr_year2 = 2022
66+ else :
67+ aatsr_year2 = end_date .year
68+ slstr_year2 = end_date .year
69+
70+ # =============================
71+ # Download AATSR data from CEDA
72+ # =============================
73+
74+ if start_date is None :
75+ start_date = datetime .datetime (aatsr_year1 , 1 , 1 , tzinfo = datetime .UTC )
3776 if end_date is None :
38- end_date = datetime (2011 , 1 , 1 )
77+ end_date = datetime .datetime (aatsr_year2 , 12 , 31 , tzinfo = datetime .UTC )
78+
3979 loop_date = start_date
4080
4181 downloader = CCIDownloader (
@@ -46,12 +86,144 @@ def download_dataset(
4686 )
4787 downloader .connect ()
4888
89+ name = "AATSR"
90+ algorithm = "SU"
91+ version = "v4.3"
92+
93+ # download monthly AATSR data
94+
95+ loop_date = start_date
4996 while loop_date <= end_date :
50- year = loop_date .year
51- if year < 2003 :
52- downloader .set_cwd ("ATSR2_SU/L3/v4.21/MONTHLY" )
97+ instrument = "ATSR2" if loop_date .year < 2003 else "AATSR"
98+ rel_base_dir = f"{ instrument } { algorithm } { version }
99+ downloader .set_cwd (rel_base_dir )
100+ if downloader .exists (f"{ loop_date .year } ):
101+ downloader .download_folder (
102+ f"{ loop_date .year } ,
103+ f"{ name } { algorithm } { version } ,
104+ )
53105 else :
54- downloader .set_cwd ("AATSR_SU/L3/v4.21/MONTHLY" )
55-
56- downloader .download_year (loop_date .year )
106+ logger .info ("%d: no data found" , loop_date .year )
57107 loop_date += relativedelta .relativedelta (years = 1 )
108+
109+ # download daily AATSR data
110+
111+ loop_date = start_date
112+ while loop_date <= end_date :
113+ instrument = "ATSR2" if loop_date .year < 2003 else "AATSR"
114+ rel_base_dir = f"{ instrument } { algorithm } { version }
115+ downloader .set_cwd (rel_base_dir )
116+ if downloader .exists (f"{ loop_date .year } ):
117+ downloader .set_cwd (f"{ rel_base_dir } { loop_date .year } )
118+ if downloader .exists (f"{ loop_date .month :02} ):
119+ downloader .download_folder (
120+ f"{ loop_date .month :02} ,
121+ f"{ name } { algorithm } { version } ,
122+ )
123+ else :
124+ logger .info (
125+ "%d/%d: no data found" ,
126+ loop_date .year ,
127+ loop_date .month ,
128+ )
129+ else :
130+ logger .info ("%d: no data found" , loop_date .year )
131+ loop_date += relativedelta .relativedelta (months = 1 )
132+
133+ # ================================================
134+ # Download SLSTR data from CDS (daily and monthly)
135+ # ================================================
136+
137+ raw_obs_dir = original_data_dir
138+ output_folder = raw_obs_dir / f"Tier{ dataset_info ['tier' ]} / dataset
139+ output_folder .mkdir (parents = True , exist_ok = True )
140+
141+ cds_url = "https://cds.climate.copernicus.eu/api"
142+
143+ name = "SLSTR"
144+ algorithm = "SU"
145+ version = "v1.12"
146+
147+ requests = {}
148+
149+ # The CDS requests have to be done for each month separately to avoid the
150+ # error "cost limits exceeded - Your request is too large, please
151+ # reduce your selection.".
152+ # Note: AOD and fine mode AOD are both selected with
153+ # "variable": "aerosol_optical_depth", so no need to have two separate
154+ # requests for "aerosol_optical_depth" and
155+ # "fine_mode_aerosol_optical_depth"
156+
157+ for year in range (slstr_year1 , slstr_year2 ):
158+ for month in range (1 , 13 ):
159+ requests .update (
160+ {
161+ "aod_slstr_daily_" + str (year ) + f"{ month :02d} : {
162+ "time_aggregation" : "daily_average" ,
163+ "variable" : "aerosol_optical_depth" ,
164+ "sensor_on_satellite" : [
165+ "slstr_on_sentinel_3a" ,
166+ "slstr_on_sentinel_3b" ,
167+ ],
168+ "algorithm" : ["swansea" ],
169+ "year" : str (year ),
170+ "month" : f"{ month :02d} ,
171+ "day" : [f"{ m :02d} for m in range (1 , 32 )],
172+ "version" : ["v1_12" ],
173+ },
174+ "aod_slstr_monthly_" + str (year ) + f"{ month :02d} : {
175+ "time_aggregation" : "monthly_average" ,
176+ "variable" : "aerosol_optical_depth" ,
177+ "sensor_on_satellite" : [
178+ "slstr_on_sentinel_3a" ,
179+ "slstr_on_sentinel_3b" ,
180+ ],
181+ "algorithm" : ["swansea" ],
182+ "year" : str (year ),
183+ "month" : f"{ month :02d} ,
184+ "version" : ["v1_12" ],
185+ },
186+ },
187+ )
188+
189+ cds_client = cdsapi .Client (cds_url )
190+
191+ for var_name , request in requests .items ():
192+ if "daily" in var_name :
193+ outdir = output_folder / f"{ name } { algorithm } { version }
194+ else :
195+ outdir = output_folder / f"{ name } { algorithm } { version }
196+ outdir .mkdir (parents = True , exist_ok = True )
197+
198+ logger .info ("Downloading %s data to %s" , var_name , outdir )
199+
200+ file_path = outdir / f"{ var_name }
201+
202+ if file_path .exists () and not overwrite :
203+ logger .info (
204+ "File %s already exists. Skipping download." ,
205+ file_path ,
206+ )
207+ continue
208+
209+ try :
210+ cds_client .retrieve (
211+ "satellite-aerosol-properties" ,
212+ request ,
213+ file_path .as_posix (),
214+ )
215+ # Handle both .gz and .zip files
216+ with open (file_path , "rb" ) as file :
217+ magic = file .read (2 )
218+
219+ if magic == b"PK" : # ZIP file signature
220+ logger .info ("Detected ZIP file: %s" , file_path )
221+ with zipfile .ZipFile (file_path , "r" ) as zip_ref :
222+ zip_ref .extractall (outdir )
223+ else :
224+ logger .info ("Detected GZIP file: %s" , file_path )
225+ with gzip .open (file_path , "rb" ) as f_in :
226+ with open (outdir / file_path .stem , "wb" ) as f_out :
227+ shutil .copyfileobj (f_in , f_out )
228+ except Exception as ex :
229+ logger .info ("%s: no data downloaded for %s" , type (ex ), var_name )
0 commit comments