697.workflow checkout tool

fre/__init__.py

@@ -9,7 +9,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s"
+FORMAT = "[%(levelname)7s:%(filename)24s:%(funcName)24s] %(message)s"
 logging.basicConfig(level = logging.WARNING,
                     format = FORMAT,
                     filename = None,

fre/fre.py

@@ -22,7 +22,8 @@
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+    lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli",
+                       "pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
                        "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",

fre/tests/test_fre_cmor_cli.py

@@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog():
     assert result.exit_code == 0
     assert Path("TEST_FOO_LOG.log").exists()
 
-    log_text_line_1='[ INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
-    log_text_line_2='[DEBUG:                  fre.py:                     fre] click entry-point function call done.\n' # pylint: disable=line-too-long
+    log_text_line_1='[   INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
+    log_text_line_2='[  DEBUG:                  fre.py:                     fre] click entry-point function call done.\n' # pylint: disable=line-too-long
     with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text:
         line_list=log_text.readlines()
         assert log_text_line_1 in line_list[0]
@@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog():
     assert result.exit_code == 0
     assert Path("TEST_FOO_LOG.log").exists()
 
-    log_text_line_1='[ INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
+    log_text_line_1='[   INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
     with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text:
         line_list=log_text.readlines()
         assert log_text_line_1 in line_list[0]

fre/tests/test_fre_workflow_cli.py

@@ -0,0 +1,76 @@
+"""
+CLI Tests for fre workflow *
+
+Tests the command-line-interface commands for each tool in the fre workflow suite.
+  - successful invocation of fre workflow $tool 
+  - successful invocation of fre workflow $tool --help
+  - expected failure for fre workflow $tool --optionDne  (failure for undefined click option)
+"""
+from pathlib import Path
+from click.testing import CliRunner
+from fre import fre
+
+runner = CliRunner()
+
+#-- fre workflow
+def test_cli_fre_workflow():
+    ''' fre workflow '''
+    result = runner.invoke(fre.fre, args=["workflow"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_help():
+    ''' fre workflow --help '''
+    result = runner.invoke(fre.fre, args=["workflow", "--help"])
+    assert result.exit_code == 0
+
+def test_cli_fre_workflow_opt_dne():
+    ''' fre workflow optionDNE '''
+    result = runner.invoke(fre.fre, args=["workflow", "optionDNE"])
+    assert result.exit_code == 2
+
+#-- fre workflow checkout
+def test_cli_fre_workflow_checkout():
+    ''' fre workflow checkout'''
+    result = runner.invoke(fre.fre, args=["workflow", "checkout"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_checkout_help():
+    ''' fre workflow checkout --help '''
+    result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"])
+    assert result.exit_code == 0
+
+def test_cli_fre_workflow_checkout_opt_dne():
+    ''' fre workflow checkout optionDNE '''
+    result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_checkout_target_dir_set(tmp_path):
+    """
+    Test checkout in target directory if --target-dir is explicitly set.
+    """
+    experiment = "c96L65_am5f7b12r1_amip_TESTING"
+    result = runner.invoke(fre.fre, args=["workflow", "checkout",
+                                          "--yamlfile", "fre/workflow/tests/AM5_example/am5.yaml",
+                                          "--experiment", experiment,
+                                          "--application", "pp",
+                                          "--target-dir", tmp_path])
+    assert result.exit_code == 0
+    assert Path(f"{tmp_path}/cylc-src/{experiment}").exists()
+
+def test_cli_fre_workflow_checkout_default_dir(monkeypatch, tmp_path):
+    """
+    Test workflow repository is cloned in the default location
+    if --target-dir is not set; default = ~./fre-workflows
+    """
+    # Create and set a mock HOME
+    fake_home = f"{tmp_path}/fake_home"
+    Path(fake_home).mkdir(parents=True,exist_ok=True)
+    monkeypatch.setenv("HOME", f"{tmp_path}/fake_home")
+
+    experiment = "c96L65_am5f7b12r1_amip_TESTING"
+    result = runner.invoke(fre.fre, args=["workflow", "checkout",
+                                          "-y", "fre/workflow/tests/AM5_example/am5.yaml",
+                                          "-e", experiment,
+                                          "-a", "pp"])
+    assert result.exit_code == 0
+    assert Path(f"{fake_home}/.fre-workflows/cylc-src/{experiment}").exists()

fre/workflow/README.md

@@ -0,0 +1,22 @@
+# FRE workflow
+
+The`fre workflow` toolset allows users to clone, install, and run a cylc workflow.
+
+The workflow repository and version are specified in `the setting.yaml`.
+
+## Quickstart
+From the top-level dircetory of the fre-cli repository:
+```
+# Checkout/clone the post-processing workflow repository
+fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING --application pp
+```
+
+## Subtools
+- `fre workflow checkout [options]`
+   - Purpose: Clone the specified workflow repository from the settings.yaml, associated with the application passed.
+   - Options:
+        - `-y, --yamlfile [model yaml] (str; required)`
+        - `-e, --experiment [experiment name] (str; required)`
+        - `-a, --application [ run | pp ] (str; required)`
+        - `--target-dir [target location where workflow will be cloned] (str; optional; default is ~/.fre-workflows`
+        - `--force-checkout (bool; optional)`

fre/workflow/checkout_script.py

@@ -0,0 +1,205 @@
+""" Workflow checkout """
+import os
+import subprocess
+import filecmp
+from pathlib import Path
+import logging
+import shutil
+from datetime import datetime
+from typing import Optional
+import json
+from jsonschema import validate, SchemaError, ValidationError
+
+import fre.yamltools.combine_yamls_script as cy
+from fre.app.helpers import change_directory
+
+fre_logger = logging.getLogger(__name__)
+
+######VALIDATE#####
+def validate_yaml(yamlfile: dict, application: str):
+    """
+    Validate the format of the yaml file against the
+    schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas).
+
+    :param yamlfile: Dictionary containing the combined model,
+                     settings, pp, and analysis yaml content
+    :type yamlfile: dict
+    :param application: type of workflow to check out/clone 
+    :type application: string
+    :raises ValueError:
+        - invalid gfdl_msd_schema path
+        - invalid combined yaml
+        - miscellaneous error in validation
+    """
+    schema_dir = Path(__file__).resolve().parents[1]
+    schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json')
+    fre_logger.info("Using yaml schema '%s'", schema_path)
+    # Load the json schema: .load() (vs .loads()) reads and parses the json in one)
+    try:
+        with open(schema_path,'r', encoding='utf-8') as s:
+            schema = json.load(s)
+    except:
+        fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path)
+        raise
+
+    # Validate yaml
+    # If the yaml is not valid, the schema validation will raise errors and exit
+    try:
+        validate(instance = yamlfile,schema=schema)
+        fre_logger.info(" ** COMBINED YAML VALID ** ")
+    except SchemaError as exc:
+        raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc
+    except ValidationError as exc:
+        raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc
+    except Exception as exc:
+        raise ValueError("Miscellaneous error from validation. Please try to find the error and try again.") from exc
+
+def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: str = None,
+                      application: str = None, force_checkout: Optional[bool] = False):
+    """
+    Create a directory and clone the workflow template files from a specified repository.
+
+    :param yamlfile: Model yaml configuration file
+    :type yamlfile: str
+    :param experiment: One of the experiment names listed in the model yaml file.
+                       Note: the command "fre list exps -y [model_yamlfile]" can be used to
+                       list the available experiment names
+    :type experiment: str
+    :param application: String used to specify the type of workflow to be cloned.
+                        Ex.: run, postprocess
+    :type application: str
+    :param target_dir: Target location to create the cylc-src/<workflow> directory in
+    :type target_dir: str
+    :param force_checkout: If the workflow directory exists, move it to an archived location
+                           (~/.fre-workflows/archived) and re-clone the workflow repository
+    :type force_checkout: bool
+    :raises OSError: if the checkout script cannot be created
+    :raises ValueError:
+        - if the repository and/or tag was not defined
+        - if the target directory does not exist or cannot be found
+        - if tag or branch does not match the git clone branch arg
+    """
+    # Used in consolidate_yamls function for now
+    platform = None
+    target = None
+
+    # Set the default target directory location
+    if target_dir is None:
+        target_dir = os.path.expanduser("~/.fre-workflows")
+
+    if application in ["run", "pp"]:
+        fre_logger.info(" ** Configuring the resolved YAML for the %s **", application)
+        yaml = cy.consolidate_yamls(yamlfile=yamlfile,
+                                    experiment=experiment,
+                                    platform=platform,
+                                    target=target,
+                                    use=application,
+                                    output="config.yaml")
+
+        validate_yaml(yamlfile = yaml, application = application)
+
+        # Reset application for pp to make it discoverable in yaml config
+        if application == "pp":
+            application = "postprocess"
+
+        workflow_info = yaml.get(application).get("workflow")
+
+    yaml_filepath = f"{Path.cwd()}/config.yaml"
+    repo = workflow_info.get("repository")
+    tag = workflow_info.get("version")
+    fre_logger.info("Defined tag ==> '%s'", tag)
+
+    if None in [repo, tag]:
+        raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}")
+
+    fre_logger.info("(%s):(%s) check out for %s ==> REQUESTED", repo, tag, application)
+
+    # Create src_dir if it does not exist
+    if not Path(target_dir).exists():
+        Path(target_dir).mkdir(parents=True, exist_ok=True)
+
+    # Define cylc-src directory
+    src_dir = f"{target_dir}/cylc-src"
+    # workflow name
+    workflow_name = experiment
+
+    # create workflow in cylc-src
+    try:
+        Path(src_dir).mkdir(parents=True, exist_ok=True)
+    except Exception as exc:
+        raise OSError(
+            f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc
+
+    if Path(f"{src_dir}/{workflow_name}").is_dir():
+        fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name)
+        if force_checkout:
+            # Create archived workflows location
+            archived = f"{target_dir}/archived_workflows"
+            Path(archived).mkdir(parents=True, exist_ok=True)
+
+            # Move previous workflow to archived location
+            fre_logger.warning(" *** Moving previous checkout to %s ***", archived)
+            shutil.move(f"{src_dir}/{workflow_name}", archived)
+
+            # Rename previous workflow
+            move_timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+            os.rename(f"{archived}/{workflow_name}", f"{archived}/{workflow_name}_{move_timestamp}")
+
+#            # Keeping this here in case we want to switch force_checkout to removing instead of moving
+#            fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name)
+#            shutil.rmtree(f"{src_dir}/{workflow_name}")
+        else:
+            with change_directory(f"{src_dir}/{workflow_name}"):
+                ## Compare previous workflow directory
+                # capture the branch and tag
+                # if either match git_clone_branch_arg, then success. otherwise, fail.
+                current_tag = subprocess.run(["git","describe","--tags"],
+                                             capture_output = True,
+                                             text = True, check = True).stdout.strip()
+                current_branch = subprocess.run(["git", "branch", "--show-current"],
+                                                 capture_output = True,
+                                                 text = True, check = True).stdout.strip()
+
+                if tag in (current_tag, current_branch):
+                    fre_logger.info("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag)
+                else:
+                    fre_logger.error(
+                        "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag)
+                    fre_logger.error(
+                        "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag)
+                    raise ValueError('Neither tag nor branch matches the git clone branch arg')
+
+
+                ## Compare content of current and previous configured, resolved yamls
+                if filecmp.cmp(yaml_filepath, "config.yaml", shallow=False):
+                    fre_logger.info("Resolved yaml already exists and did not change.")
+                else:
+                    fre_logger.error("")
+                    fre_logger.error("ERROR: Checkout and resolved yaml already exist but resolved yaml files "
+                                     "are not identical!")
+                    fre_logger.error("For troubleshooting:")
+                    fre_logger.error("  - Current resolved yaml: %s", yaml_filepath)
+                    fre_logger.error("  - Previous resolved yaml: %s", f"{src_dir}/{workflow_name}/config.yaml")
+                    fre_logger.error("Try:")
+                    fre_logger.error("  - resolving yaml differences if nothing else has changed")
+                    fre_logger.error(f"  - removing the {target_dir}/cylc-src/{workflow_name} folder and "
+                                           "re-running the command")
+                    fre_logger.error("  - pass the --force-checkout option to archive the workflow (move to "
+                                     "~/.fre-workflows/archived/) and clone a new workflow.")
+                    return
+#                    raise ValueError("Resolve yaml differences or pass --force-checkout to archive the workflow"
+#                                     "(moved to ~/.fre-workflows/archived) and clone a new workflow.")
+
+    if not Path(f"{src_dir}/{workflow_name}").is_dir():
+        fre_logger.info("Workflow does not exist; will create now")
+        clone_output = subprocess.run( ["git", "clone","--recursive",
+                                        f"--branch={tag}",
+                                        repo, f"{src_dir}/{workflow_name}"],
+                                        capture_output = True, text = True, check = True)
+        fre_logger.debug(clone_output)
+        fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag)
+
+        ## Move combined yaml to cylc-src location
+        current_dir = Path.cwd()
+        shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}")
+        fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name)

fre/workflow/freworkflow.py

@@ -0,0 +1,40 @@
+''' fre workflow click interface for fre workflow subcommands'''
+import os
+import click
+import logging
+fre_logger = logging.getLogger(__name__)
+
+#fre tools
+from . import checkout_script
+#from . import install_script
+#from . import run_script
+
+@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210)))
+def workflow_cli():
+    ''' entry point to fre workflow click commands '''
+
+@workflow_cli.command()
+@click.option("-y", "--yamlfile", type=str,
+              help="Model yaml file",
+              required=True)
+@click.option("-e", "--experiment", type=str,
+              help="Experiment name",
+              required=True)
+@click.option("-a", "--application",
+              type=click.Choice(['run', 'pp']),
+              help="Type of workflow to check out/clone",
+              required=True)
+@click.option("--target-dir",
+              type=str,
+              help=f"""Target directory for the workflow to be cloned into.
+                    If not defined, a default location of ~/.fre-workflows
+                    will be used""")
+@click.option("--force-checkout",
+              is_flag=True,
+              default=False,
+              help="If the checkout already, exists, remove and clone the desired repo again.")
+def checkout(target_dir, yamlfile, experiment, application, force_checkout):
+    """
+    Checkout/clone the workflow repository.
+    """
+    checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout)