Merge pull request #5 from CMCC-Foundation/no-shell

Removed shell equals true from the reported files

Author: atakeigor

main.py

@@ -10,7 +10,7 @@
 import pandas as pd
 from datetime import datetime
 from argparse import ArgumentParser
-from glob import glob
+from glob import glob as gg
 
 # Import medslik modules
 from src.utils import Utils, Config, read_oilbase
@@ -183,16 +183,13 @@ def data_download_medslik(
                 password=copernicus_pass,
             )
 
-            subprocess.run(
-                [
-                    f'cp {output_path}*{identifier}*{config["simulation"]["name"]}*.nc {root_directory}/oce_files/'
-                ],
-                shell=True,
-            )
-            subprocess.run(
-                [f'rm {output_path}*{identifier}*{config["simulation"]["name"]}*.nc'],
-                shell=True,
-            )
+            source_files = gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc")
+            destination = os.path.join(root_directory, "oce_files")
+            for file in source_files:
+                shutil.copy(file, destination)
+
+            for file in gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc"):
+                os.remove(file)
 
         if config["download"]["download_wind"]:
             # ensuring .cdsapirc is created in the home directory
@@ -216,16 +213,13 @@ def data_download_medslik(
             )
             process_era5(output_path=output_path, output_name=output_name)
 
-            subprocess.run(
-                [
-                    f'cp {output_path}*{identifier}*{config["simulation"]["name"]}*.nc {root_directory}/met_files/'
-                ],
-                shell=True,
-            )
-            subprocess.run(
-                [f'rm {output_path}*{identifier}*{config["simulation"]["name"]}*.nc'],
-                shell=True,
-            )
+            source_files = gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc")
+            destination = os.path.join(root_directory, "met_files")
+            for file in source_files:
+                shutil.copy(file, destination)
+
+            for file in gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc"):
+                os.remove(file)
 
     def run_preproc(config: dict, exp_folder: str, lon_min, lon_max, lat_min, lat_max):
         """
@@ -343,98 +337,75 @@ def run_medslik_sim(self, simdir, simname, separate_slicks=False):
 
         output_dir = f"{model_dir}OUT/MDK_SIM_{year}_{month:02d}_{day:02d}_{hour:02d}{minute:02d}_{simname}/."
 
-        # removing old outputes just to be sure
-        subprocess.run([f"rm -rf {output_dir}"], shell=True)
-
-        if separate_slicks == False:
-            # copy METOCEAN files to MEDSLIK-II installation
-            subprocess.run(
-                [f"cp {simdir}{simname}/oce_files/*.mrc {model_dir}RUN/TEMP/OCE/"],
-                shell=True,
-                check=True,
-            )
-            subprocess.run(
-                [f"cp {simdir}{simname}/met_files/*.eri {model_dir}RUN/TEMP/MET/"],
-                shell=True,
-                check=True,
-            )
-            # copy bnc files
-            subprocess.run(
-                [f"cp {simdir}{simname}/bnc_files/* {model_dir}DTM_INP/"],
-                shell=True,
-                check=True,
-            )
-            # copy Extract and config files
-            subprocess.run(
-                [
-                    f"cp {simdir}{simname}/xp_files/medslik_II.for {model_dir}RUN/MODEL_SRC/"
-                ],
-                shell=True,
-                check=True,
-            )
-            subprocess.run(
-                [f"cp {simdir}{simname}/xp_files/config2.txt {model_dir}RUN/"],
-                shell=True,
-                check=True,
-            )
-            subprocess.run(
-                [f"cp {simdir}{simname}/xp_files/config1.txt {model_dir}RUN/"],
-                shell=True,
-                check=True,
-            )
-            # Compile and start running
-            subprocess.run(
-                [f"cd {model_dir}RUN/; sh MODEL_SRC/compile.sh; ./RUN.sh"],
-                shell=True,
-                check=True,
-            )
+        # Remove old outputs (equivalent to `rm -rf`)
+        if os.path.exists(output_dir):
+            shutil.rmtree(output_dir)
+
+        if not separate_slicks:
+            # Copy METOCEAN files
+            oce_files = gg(f"{simdir}{simname}/oce_files/*.mrc")
+            met_files = gg(f"{simdir}{simname}/met_files/*.eri")
+            bnc_files = gg(f"{simdir}{simname}/bnc_files/*")
+            xp_files = {
+                "medslik_II.for": os.path.join(simdir, simname, "xp_files", "medslik_II.for"),
+                "config2.txt": os.path.join(simdir, simname, "xp_files", "config2.txt"),
+                "config1.txt": os.path.join(simdir, simname, "xp_files", "config1.txt"),
+            }
+
+            # Copy METOCEAN, MET, and BNC files
+            for file in oce_files:
+                shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "OCE"))
+            for file in met_files:
+                shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "MET"))
+            for file in bnc_files:
+                shutil.copy(file, os.path.join(model_dir, "DTM_INP"))
+
+            # Copy other required files
+            for dest, src in xp_files.items():
+                shutil.copy(src, os.path.join(model_dir, "RUN", dest if "config" in dest else "MODEL_SRC"))
+
+            # Compile and start running (replacing `cd` with `cwd`)
+            compile_script = "MODEL_SRC/compile.sh"
+            run_script = "RUN.sh"
+            subprocess.run(["sh", compile_script], check=True, cwd=os.path.join(model_dir, "RUN"))
+            subprocess.run(["./" + run_script], check=True, cwd=os.path.join(model_dir, "RUN"))
 
         else:
-            slicks = glob(f"{simdir}{simname}/xp_files/*/")
-            for i in range(0, len(slicks)):
-                subprocess.run(
-                    [f"cp {simdir}{simname}/oce_files/*.mrc {model_dir}RUN/TEMP/OCE/"],
-                    shell=True,
-                )
-                subprocess.run(
-                    [f"cp {simdir}{simname}/met_files/*.eri {model_dir}RUN/TEMP/MET/"],
-                    shell=True,
-                )
-                # copy bnc files
-                subprocess.run(
-                    [f"cp {simdir}{simname}/bnc_files/* {model_dir}DTM_INP/"],
-                    shell=True,
-                )
-                # copy Extract and config files
-                subprocess.run(
-                    [
-                        f"cp {simdir}{simname}/xp_files/medslik_II.for {model_dir}RUN/MODEL_SRC/"
-                    ],
-                    shell=True,
-                )
-                subprocess.run(
-                    [f"cp {simdir}{simname}/xp_files/config2.txt {model_dir}RUN/"],
-                    shell=True,
-                )
-                subprocess.run(
-                    [
-                        f"cp {simdir}{simname}/xp_files/slick{i+1}/config1.txt {model_dir}RUN/"
-                    ],
-                    shell=True,
-                )
-                # Compile and start running
-                subprocess.run(
-                    [f"cd {model_dir}RUN/; sh MODEL_SRC/compile.sh; ./RUN.sh"],
-                    shell=True,
-                    check=True,
-                )
+            # Handle separate slicks
+            slicks = gg(f"{simdir}{simname}/xp_files/*/")
+            for i, slick_dir in enumerate(slicks):
+                # Copy METOCEAN, MET, and BNC files
+                oce_files = gg(f"{simdir}{simname}/oce_files/*.mrc")
+                met_files = gg(f"{simdir}{simname}/met_files/*.eri")
+                bnc_files = gg(f"{simdir}{simname}/bnc_files/*")
+                config1_path = os.path.join(simdir, simname, f"xp_files/slick{i + 1}/config1.txt")
+
+                for file in oce_files:
+                    shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "OCE"))
+                for file in met_files:
+                    shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "MET"))
+                for file in bnc_files:
+                    shutil.copy(file, os.path.join(model_dir, "DTM_INP"))
+                shutil.copy(config1_path, os.path.join(model_dir, "RUN", "config1.txt"))
 
-        # Send files to case dir and remove temp files
-        subprocess.run([f"cp -r {output_dir} {simdir}{simname}/out_files/"], shell=True)
-        subprocess.run(
-            [f"rm -rf {simdir}{simname}/out_files/MET {simdir}{simname}/out_files/OCE"],
-            shell=True,
-        )
+                # Compile and start running
+                compile_script = "MODEL_SRC/compile.sh"
+                run_script = "RUN.sh"
+                subprocess.run(["sh", compile_script], check=True, cwd=os.path.join(model_dir, "RUN"))
+                subprocess.run(["./" + run_script], check=True, cwd=os.path.join(model_dir, "RUN"))
+
+        # Copy output files (replacing `cp -r`)
+        output_dest = os.path.join(simdir, simname, "out_files")
+        if os.path.exists(output_dir):
+            shutil.copytree(output_dir, output_dest, dirs_exist_ok=True)
+
+        # Remove temporary MET and OCE files (equivalent to `rm -rf`)
+        temp_met = os.path.join(output_dest, "MET")
+        temp_oce = os.path.join(output_dest, "OCE")
+        if os.path.exists(temp_met):
+            shutil.rmtree(temp_met)
+        if os.path.exists(temp_oce):
+            shutil.rmtree(temp_oce)
 
 
 if __name__ == "__main__":

src/download/download_copernicus_parser.py

@@ -74,7 +74,9 @@ def download_copernicus(
         ds.to_netcdf(output_name)
 
         # remove the temporary files
-        subprocess.run([f"rm -rf {output_path}temp.nc"], shell=True)
+        temp_file = os.path.join(output_path, "temp.nc")
+        if os.path.exists(temp_file):
+            os.remove(temp_file)
 
     else:
 
@@ -150,6 +152,7 @@ def download_copernicus(
         ds.to_netcdf(output_name)
 
         # remove the temporary files
-        subprocess.run(
-            [f"rm -rf {output_path}/curr.nc {output_path}/temp.nc"], shell=True
-        )
+        temp_files = [os.path.join(output_path, "curr.nc"), os.path.join(output_path, "temp.nc")]
+        for temp_file in temp_files:
+            if os.path.exists(temp_file):
+                os.remove(temp_file)

src/download/download_era5_parser.py

@@ -6,6 +6,7 @@
 import argparse
 import subprocess
 import time
+from glob import glob as gg
 
 # Functions outside this script
 from WITOIL_iMagine.src.utils.utils import *
@@ -79,7 +80,10 @@ def process_era5(output_path,output_name):
     met.to_netcdf(output_name)
 
     #remove the temporary files
-    subprocess.run([f'rm -rf {output_path}/temp*.nc'],shell=True)
+    temp_files = gg(os.path.join(output_path, "temp*.nc"))
+    for temp_file in temp_files:
+        if os.path.exists(temp_file):
+            os.remove(temp_file)
 
 
 if __name__ == '__main__':
@@ -125,5 +129,4 @@ def process_era5(output_path,output_name):
 
     # os.system('rm ' + out_folder + '/output.nc')
 
-    # os.system('rm ' + out_folder + '/pre_*.nc')
-
+    # os.system('rm ' + out_folder + '/pre_*.nc')

src/plot/plot_mdk3.py

@@ -180,7 +180,7 @@ def create_gif(self):
                 f"magick -delay 20 -loop 0 {path}/*surf_oil_*.png \
                     {self.out_figures}/oil_concentration_{self.config['simulation']['name']}.gif"
             ],
-            shell=True,
+            # shell=True,
         )
 
     def plot_pyngl(
@@ -208,7 +208,7 @@ def plot_pyngl(
                                 {sim_length} {plot_lon[0]} {plot_lon[1]} \
                                 {plot_lat[0]} {plot_lat[1]}"
             ],
-            shell=True,
+            # shell=True,
             check=True,
         )
 

src/preprocessing/preprocessing_mdk3.py

@@ -219,14 +219,20 @@ def process_medslik_memmory_array(self):
 
         med_for = f'{self.exp_folder}/xp_files/medslik_II.for'
 
-        subprocess.run([f'cp WITOIL_iMagine/src/templates/medslik_II_template.for {med_for}'],shell=True)
+        # Refactored: Copying the template file
+        template_path = 'WITOIL_iMagine/src/templates/medslik_II_template.for'
+        shutil.copy(template_path, med_for)
 
-        # Replacing NMAX in medslik fortran with a python function
+        # Replacing NMAX in medslik Fortran with a Python function
         Utils.search_and_replace(med_for, 'NMAX', str(nmax))
 
     def configuration_parameters(self):
 
-        subprocess.run([f'cp WITOIL_iMagine/src/templates/config2.txt {self.exp_folder}/xp_files/config2.txt'],shell=True)
+        source_file = 'WITOIL_iMagine/src/templates/config2.txt'
+        destination_file = f'{self.exp_folder}/xp_files/config2.txt'
+
+        # Refactored: Copy the file
+        shutil.copy(source_file, destination_file)
 
     def common_grid(self):
 
@@ -271,9 +277,9 @@ def write_config_files(self,
             config_file = f"WITOIL_iMagine/cases/{simname}/xp_files/config1.txt"
         else:
             config_file = f"WITOIL_iMagine/cases/{simname}/xp_files/slick{s_num+1}/config1.txt"
-        subprocess.run(
-            [f"cp WITOIL_iMagine/src/templates/config1_template_0.txt {config_file}"], shell=True
-        )
+        # Refactored: Copy the template file
+        source_file = "WITOIL_iMagine/src/templates/config1_template_0.txt"
+        shutil.copy(source_file, config_file)
         # adding spill Name - Add slick number if separate slicks
         if separate_slicks == False:
             Utils.search_and_replace(config_file, "RUNNAME", simname)
@@ -331,4 +337,3 @@ def write_config_files(self,
 if __name__ == "__main__":
 
     pass
-