Apply automatic fixes for style issues (#4263)

Author: bouweandela

esmvaltool/__init__.py

@@ -7,7 +7,7 @@
 except PackageNotFoundError as exc:
     raise PackageNotFoundError(
         "ESMValTool package not found, please run `pip install -e .` before "
-        "importing the package."
+        "importing the package.",
     ) from exc
 
 

esmvaltool/cmorizers/data/cmorizer.py

@@ -74,7 +74,7 @@ def start(self, command, datasets, config_file, config_dir, options):
             if not config_dir.is_dir():
                 raise NotADirectoryError(
                     f"Invalid --config_dir given: {config_dir} is not an "
-                    f"existing directory"
+                    f"existing directory",
                 )
             CFG.update_from_dirs([config_dir])
         CFG.nested_update(options)
@@ -85,7 +85,8 @@ def start(self, command, datasets, config_file, config_dir, options):
 
         # configure logging
         log_files = configure_logging(
-            output_dir=self.run_dir, console_log_level=self.log_level
+            output_dir=self.run_dir,
+            console_log_level=self.log_level,
         )
         logger.info("Writing program log files to:\n%s", "\n".join(log_files))
 
@@ -175,7 +176,7 @@ def download_dataset(self, dataset, start_date, end_date, overwrite):
         """
         if not self.has_downloader(dataset):
             raise ValueError(
-                f"Dataset {dataset} does not have an automatic downloader"
+                f"Dataset {dataset} does not have an automatic downloader",
             )
         dataset_module = self._dataset_to_module(dataset)
         logger.info("Downloading %s", dataset)
@@ -232,7 +233,7 @@ def format(self, start, end, install):
 
         if failed_datasets:
             raise RuntimeError(
-                f"Format failed for datasets {' '.join(failed_datasets)}"
+                f"Format failed for datasets {' '.join(failed_datasets)}",
             )
 
     @staticmethod
@@ -327,11 +328,20 @@ def format_dataset(self, dataset, start, end, install):
         if os.path.isfile(reformat_script_root + ".ncl"):
             reformat_script = reformat_script_root + ".ncl"
             success = self._run_ncl_script(
-                in_data_dir, out_data_dir, dataset, reformat_script, start, end
+                in_data_dir,
+                out_data_dir,
+                dataset,
+                reformat_script,
+                start,
+                end,
             )
         elif os.path.isfile(reformat_script_root + ".py"):
             success = self._run_pyt_script(
-                in_data_dir, out_data_dir, dataset, start, end
+                in_data_dir,
+                out_data_dir,
+                dataset,
+                start,
+                end,
             )
         else:
             logger.error("Could not find formatter for %s", dataset)
@@ -354,15 +364,17 @@ def format_dataset(self, dataset, start, end, install):
                 )
             else:
                 logger.info(
-                    "Installing dataset %s in folder %s", dataset, target_dir
+                    "Installing dataset %s in folder %s",
+                    dataset,
+                    target_dir,
                 )
                 shutil.move(out_data_dir, target_dir)
         return True
 
     def _get_dataset_tier(self, dataset):
         for tier in [2, 3]:
             if os.path.isdir(
-                os.path.join(self.rawobs, f"Tier{tier}", dataset)
+                os.path.join(self.rawobs, f"Tier{tier}", dataset),
             ):
                 return f"Tier{tier}"
         return None
@@ -405,31 +417,41 @@ def _write_ncl_settings(
     def _run_ncl_script(self, in_dir, out_dir, dataset, script, start, end):
         """Run the NCL cmorization mechanism."""
         logger.info(
-            "CMORizing dataset %s using NCL script %s", dataset, script
+            "CMORizing dataset %s using NCL script %s",
+            dataset,
+            script,
         )
         project = {}
         project[dataset] = {}
         project[dataset]["indir"] = in_dir
         project[dataset]["outdir"] = out_dir
         settings_file = self._write_ncl_settings(
-            project, dataset, self.run_dir, script, start, end
+            project,
+            dataset,
+            self.run_dir,
+            script,
+            start,
+            end,
         )
 
         # put settings in environment
         env = dict(os.environ)
         env["settings"] = settings_file
         env["esmvaltool_root"] = os.path.dirname(
-            os.path.dirname(os.path.dirname(os.path.dirname(script)))
+            os.path.dirname(os.path.dirname(os.path.dirname(script))),
         )
         env["cmor_tables"] = str(
-            Path(esmvalcore.cmor.__file__).parent / "tables"
+            Path(esmvalcore.cmor.__file__).parent / "tables",
         )
         logger.info("Using CMOR tables at %s", env["cmor_tables"])
         # call NCL
         ncl_call = ["ncl", script]
         logger.info("Executing cmd: %s", " ".join(ncl_call))
         with subprocess.Popen(
-            ncl_call, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env
+            ncl_call,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env,
         ) as process:
             output, err = process.communicate()
         for oline in str(output.decode("utf-8")).split("\n"):
@@ -475,12 +497,13 @@ def list(self):
         print("-" * 71)
         for dataset, dataset_info in self._info["datasets"].items():
             date = datetime.datetime.strptime(
-                str(dataset_info["last_access"]), "%Y-%m-%d"
+                str(dataset_info["last_access"]),
+                "%Y-%m-%d",
             )
             print(
                 f"| {dataset:30} | {dataset_info['tier']:4} "
                 f"| {self._has_downloader(dataset):13} "
-                f"|  {date.strftime('%Y-%m-%d')} |"
+                f"|  {date.strftime('%Y-%m-%d')} |",
             )
         print("-" * 71)
 
@@ -498,7 +521,7 @@ def info(self, dataset):
         print(f"Tier: {dataset_info['tier']}")
         print(f"Source: {dataset_info['source']}")
         print(f"Automatic download: {self._has_downloader(dataset)}")
-        print("")
+        print()
         print(dataset_info["info"])
 
     def download(
@@ -549,7 +572,11 @@ def download(
         end = self._parse_date(end)
 
         self.formatter.start(
-            "download", datasets, config_file, config_dir, kwargs
+            "download",
+            datasets,
+            config_file,
+            config_dir,
+            kwargs,
         )
         self.formatter.download(start, end, overwrite)
 
@@ -601,7 +628,11 @@ def format(
         end = self._parse_date(end)
 
         self.formatter.start(
-            "formatting", datasets, config_file, config_dir, kwargs
+            "formatting",
+            datasets,
+            config_file,
+            config_dir,
+            kwargs,
         )
         self.formatter.format(start, end, install)
 
@@ -656,7 +687,11 @@ def prepare(
         end = self._parse_date(end)
 
         self.formatter.start(
-            "preparation", datasets, config_file, config_dir, kwargs
+            "preparation",
+            datasets,
+            config_file,
+            config_dir,
+            kwargs,
         )
         if self.formatter.download(start, end, overwrite):
             self.formatter.format(start, end, install)
@@ -678,6 +713,6 @@ def _parse_date(date):
             raise ValueError(
                 f"Unsupported date format for {date}. "
                 'Supported formats for "start" and "end" are: '
-                '"None", "YYYY", "YYYYMM", "YYYYMMDD"'
+                '"None", "YYYY", "YYYYMM", "YYYYMMDD"',
             )
         return datetime.datetime.strptime(date_string, format_string)

esmvaltool/cmorizers/data/download_scripts/download_era_interim.py

@@ -106,8 +106,8 @@
         "stream": "moda",
         "type": "an",
         "levelist": "1/5/10/20/30/50/70/100/150/200/"
-        + "250/300/400/500/600/700/850"
-        + "/925/1000",  # CMIP6 Amon table, plev19
+        "250/300/400/500/600/700/850"
+        "/925/1000",  # CMIP6 Amon table, plev19
     },
 }
 
@@ -180,7 +180,7 @@ def _get_land_data(params, timesteps, years, server, era_interim_land_dir):
                     "target": f"{era_interim_land_dir}/ERA-Interim-Land_{symbol}"
                     f"_{frequency}_{year}.nc",
                     **timesteps[timestep],
-                }
+                },
             )
 
 
@@ -201,7 +201,7 @@ def _get_daily_data(params, timesteps, years, server, era_interim_dir):
                     "target": f"{era_interim_dir}/ERA-Interim_{symbol}"
                     f"_{frequency}_{year}.nc",
                     **timesteps[timestep],
-                }
+                },
             )
 
 
@@ -215,7 +215,7 @@ def _get_monthly_data(params, timesteps, years, server, era_interim_dir):
                     "dataset": "interim",
                     # All months of a year eg. 19900101/.../19901101/19901201
                     "date": "/".join(
-                        [f"{year}{m:02}01" for m in range(1, 13)]
+                        [f"{year}{m:02}01" for m in range(1, 13)],
                     ),
                     "expver": "1",
                     "grid": "0.75/0.75",
@@ -224,7 +224,7 @@ def _get_monthly_data(params, timesteps, years, server, era_interim_dir):
                     "target": f"{era_interim_dir}/ERA-Interim_{symbol}"
                     f"_{frequency}_{year}.nc",
                     **timesteps[timestep],
-                }
+                },
             )
 
 
@@ -245,7 +245,7 @@ def _get_invariant_data(params, server, era_interim_dir):
                 "type": "an",
                 "format": "netcdf",
                 "target": f"{era_interim_dir}/ERA-Interim_{symbol}.nc",
-            }
+            },
         )
 
 
@@ -259,21 +259,24 @@ def cli():
         help="Config file",
     )
     parser.add_argument(
-        "--start_year", type=int, default=1979, help="Start year"
+        "--start_year",
+        type=int,
+        default=1979,
+        help="Start year",
     )
     parser.add_argument("--end_year", type=int, default=2019, help="End year")
     args = parser.parse_args()
 
     # get and read config file
     config_file_name = os.path.abspath(
-        os.path.expandvars(os.path.expanduser(args.config_file))
+        os.path.expandvars(os.path.expanduser(args.config_file)),
     )
 
     with open(config_file_name) as config_file:
         config = yaml.safe_load(config_file)
 
     rawobs_dir = os.path.abspath(
-        os.path.expandvars(os.path.expanduser(config["rootpath"]["RAWOBS"]))
+        os.path.expandvars(os.path.expanduser(config["rootpath"]["RAWOBS"])),
     )
     era_interim_dir = f"{rawobs_dir}/Tier3/ERA-Interim"
     os.makedirs(era_interim_dir, exist_ok=True)
@@ -285,11 +288,19 @@ def cli():
 
     _get_daily_data(DAY_PARAMS, DAY_TIMESTEPS, years, server, era_interim_dir)
     _get_monthly_data(
-        MONTH_PARAMS, MONTH_TIMESTEPS, years, server, era_interim_dir
+        MONTH_PARAMS,
+        MONTH_TIMESTEPS,
+        years,
+        server,
+        era_interim_dir,
     )
     _get_invariant_data(INVARIANT_PARAMS, server, era_interim_dir)
     _get_land_data(
-        LAND_PARAMS, DAY_TIMESTEPS, years, server, era_interim_land_dir
+        LAND_PARAMS,
+        DAY_TIMESTEPS,
+        years,
+        server,
+        era_interim_land_dir,
     )
 
 

esmvaltool/cmorizers/data/downloaders/cds.py

@@ -52,15 +52,20 @@ def __init__(
                 logger.error(
                     "Could not connect to the CDS due to issues with your "
                     '".cdsapirc" file. More info in '
-                    "https://cds.climate.copernicus.eu/api-how-to."
+                    "https://cds.climate.copernicus.eu/api-how-to.",
                 )
             raise
         self._product_name = product_name
         self._request_dict = request_dictionary
         self.extra_name = extra_name
 
     def download(
-        self, year, month, day=None, file_pattern=None, file_format="tar"
+        self,
+        year,
+        month,
+        day=None,
+        file_pattern=None,
+        file_format="tar",
     ):
         """Download a specific month from the CDS.
 
@@ -139,7 +144,8 @@ def download_request(self, filename, request=None):
                 os.remove(filename)
             else:
                 logger.info(
-                    "File %s already downloaded. Skipping...", filename
+                    "File %s already downloaded. Skipping...",
+                    filename,
                 )
                 return
         try:

esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py

@@ -8,7 +8,12 @@
 
 
 def download_dataset(
-    config, dataset, dataset_info, start_date, end_date, overwrite
+    config,
+    dataset,
+    dataset_info,
+    start_date,
+    end_date,
+    overwrite,
 ):
     """Download dataset.