Skip to content

Commit 8e6f9f8

Browse files
tausiajtausiaj
authored
Merge pull request #21 from GeoOcean/feature/downloaders
Feature/downloaders
2 parents 344ddbb + 205c378 commit 8e6f9f8

File tree

2 files changed

+86
-40
lines changed

2 files changed

+86
-40
lines changed

‎bluemath_tk/downloaders/copernicus/ERA5/ERA5_download.py‎

100644100755
File mode changed.

‎bluemath_tk/downloaders/copernicus/copernicus_downloader.py‎

Lines changed: 86 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@
88

99
config = {
1010
"url": "https://cds.climate.copernicus.eu/api", # /v2?
11-
"key": "your-api-token",
11+
"key": "5cf7efae-13fc-4085-8a98-80d82bdb55f5",
1212
}
1313

14+
# javi: 5cf7efae-13fc-4085-8a98-80d82bdb55f5
15+
# valva: 45c9b669-4fbc-4d9b-9386-8c2c611a3c93
16+
# laura: c17382de-3363-47ea-9faa-ba4a9cce66b8
17+
1418

1519
class CopernicusDownloader(BlueMathDownloader):
1620
"""
@@ -33,7 +37,8 @@ class CopernicusDownloader(BlueMathDownloader):
3337

3438
products_configs = {
3539
"ERA5": json.load(
36-
open(os.path.join(os.path.dirname(__file__), "ERA5", "ERA5_config.json"))
40+
# open(os.path.join(os.path.dirname(__file__), "ERA5", "ERA5_config.json"))
41+
open("/home/grupos/geocean/tausiaj/BlueMath_tk/bluemath_tk/downloaders/copernicus/ERA5/ERA5_config.json")
3742
)
3843
}
3944

@@ -154,7 +159,7 @@ def download_data_era5(
154159
day: List[str] = None,
155160
time: List[str] = None,
156161
product_type: str = "reanalysis",
157-
data_format: str = "netcdf",
162+
data_format: str = "netcdf_legacy",
158163
download_format: str = "unarchived",
159164
force: bool = False,
160165
) -> str:
@@ -235,6 +240,7 @@ def download_data_era5(
235240

236241
fully_downloaded_files: List[str] = []
237242
NOT_fullly_downloaded_files: List[str] = []
243+
error_files: List[str] = []
238244

239245
for variable in variables:
240246
for year in years:
@@ -295,36 +301,71 @@ def download_data_era5(
295301
# Create the output directory if it does not exist
296302
os.makedirs(os.path.dirname(output_nc_file), exist_ok=True)
297303

298-
if self.check or not force:
299-
if os.path.exists(output_nc_file):
300-
self.logger.debug(
301-
f"Checking {output_nc_file} file is complete"
302-
)
303-
nc = xr.open_dataset(output_nc_file)
304-
_, last_day = calendar.monthrange(int(year), int(month))
305-
last_hour = f"{year}-{int(month):02d}-{last_day}T23"
306-
last_hour_nc = str(nc.valid_time[-1].values)
307-
nc.close()
308-
if last_hour not in last_hour_nc:
304+
self.logger.info(f"""
305+
306+
Analyzing {output_nc_file}
307+
308+
""")
309+
310+
try:
311+
312+
if self.check or not force:
313+
if os.path.exists(output_nc_file):
309314
self.logger.debug(
310-
f"{output_nc_file} ends at {last_hour_nc} instead of {last_hour}"
315+
f"Checking {output_nc_file} file is complete"
311316
)
312-
if self.check:
313-
NOT_fullly_downloaded_files.append(output_nc_file)
314-
else:
315-
self.logger.debug(
316-
f"Downloading: {variable} to {output_nc_file} because it is not complete"
317-
)
318-
self.client.retrieve(
319-
name=variable_config["dataset"],
320-
request=template_for_variable,
321-
target=output_nc_file,
317+
try:
318+
nc = xr.open_dataset(output_nc_file)
319+
_, last_day = calendar.monthrange(int(year), int(month))
320+
last_hour = f"{year}-{int(month):02d}-{last_day}T23"
321+
last_hour_nc = str(nc.valid_time[-1].values)
322+
nc.close()
323+
if last_hour not in last_hour_nc:
324+
self.logger.debug(
325+
f"{output_nc_file} ends at {last_hour_nc} instead of {last_hour}"
326+
)
327+
if self.check:
328+
NOT_fullly_downloaded_files.append(output_nc_file)
329+
else:
330+
self.logger.debug(
331+
f"Downloading: {variable} to {output_nc_file} because it is not complete"
332+
)
333+
self.client.retrieve(
334+
name=variable_config["dataset"],
335+
request=template_for_variable,
336+
target=output_nc_file,
337+
)
338+
fully_downloaded_files.append(output_nc_file)
339+
else:
340+
fully_downloaded_files.append(output_nc_file)
341+
except Exception as e:
342+
self.logger.error(
343+
f"Error was raised opening {output_nc_file}, re-downloading..."
322344
)
323-
fully_downloaded_files.append(output_nc_file)
345+
if self.check:
346+
NOT_fullly_downloaded_files.append(output_nc_file)
347+
else:
348+
self.logger.debug(
349+
f"Downloading: {variable} to {output_nc_file} because it is not complete"
350+
)
351+
self.client.retrieve(
352+
name=variable_config["dataset"],
353+
request=template_for_variable,
354+
target=output_nc_file,
355+
)
356+
fully_downloaded_files.append(output_nc_file)
357+
elif self.check:
358+
NOT_fullly_downloaded_files.append(output_nc_file)
324359
else:
360+
self.logger.debug(
361+
f"Downloading: {variable} to {output_nc_file}"
362+
)
363+
self.client.retrieve(
364+
name=variable_config["dataset"],
365+
request=template_for_variable,
366+
target=output_nc_file,
367+
)
325368
fully_downloaded_files.append(output_nc_file)
326-
elif self.check:
327-
NOT_fullly_downloaded_files.append(output_nc_file)
328369
else:
329370
self.logger.debug(
330371
f"Downloading: {variable} to {output_nc_file}"
@@ -335,22 +376,27 @@ def download_data_era5(
335376
target=output_nc_file,
336377
)
337378
fully_downloaded_files.append(output_nc_file)
338-
else:
339-
self.logger.debug(
340-
f"Downloading: {variable} to {output_nc_file}"
341-
)
342-
self.client.retrieve(
343-
name=variable_config["dataset"],
344-
request=template_for_variable,
345-
target=output_nc_file,
346-
)
347-
fully_downloaded_files.append(output_nc_file)
379+
380+
except Exception as e:
381+
382+
self.logger.error(f"""
383+
384+
Skippping {output_nc_file} for {e}
385+
386+
""")
387+
error_files.append(output_nc_file)
388+
389+
fully_downloaded_files_str = '\n'.join(fully_downloaded_files)
390+
NOT_fullly_downloaded_files_str = '\n'.join(NOT_fullly_downloaded_files)
391+
error_files = '\n'.join(error_files)
348392

349393
return f"""
350394
Fully downloaded files:
351-
{'\n'.join(fully_downloaded_files)}
395+
{fully_downloaded_files_str}
352396
Not fully downloaded files:
353-
{'\n'.join(NOT_fullly_downloaded_files)}
397+
{NOT_fullly_downloaded_files_str}
398+
Error files:
399+
{error_files}
354400
"""
355401

356402

0 commit comments

Comments
 (0)