Add integration tests with Github Actions! (#172)

bshifaw · web-flow · commit 369fcd98cc02 · 2021-10-18T11:49:22.000-04:00
* Added new integration tests that run against a local cromwell instance running on Github actions!
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -0,0 +1,72 @@
+name: Integration Tests
+
+# Controls when the action will run.
+# Triggers the workflow on push or pull request events
+on: [push, workflow_dispatch]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [ 3.7, 3.8, 3.9 ]
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      # Using Conda to install womtool causes the default system python version
+      # to be set to a later version. Placing the Install Womtool step before
+      # Setup Python allows the tox test to run on py version specified in git matrix.
+      - name: Install Womtool
+        run: |
+          $CONDA/bin/conda install -y -c bioconda womtool
+          echo "$CONDA/bin" >> $GITHUB_PATH
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install Dev Requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+
+      - name: Pull and setup mysql dockers
+        # Parameters set in the command below must be reused in the cromwell
+        # config file: tests/resources/cromwell_application.conf
+        run: |
+          docker run \
+          -p 3306:3306 \
+          --ip=172.17.0.2 \
+          --name NameOfTheContainer \
+          -e MYSQL_ROOT_PASSWORD=YourPassword \
+          -e MYSQL_DATABASE=DatabaseName \
+          -e MYSQL_USER=ChooseAName \
+          -e MYSQL_PASSWORD=YourOtherPassword \
+          -d \
+          mysql/mysql-server:5.5
+
+      - name: Pull and setup cromwell docker
+        run: |
+          export CROMWELL_TAG=$(curl -s https://api.github.com/repos/broadinstitute/cromwell/releases/latest | \
+          grep -o '"tag_name": "[^"]*' | \
+          grep -o '[^"]*$' | \
+          grep -Eo "^[0-9]+")
+
+          docker run \
+          -p 8000:8000 \
+          -v ${PWD}/tests/resources/cromwell_application.conf:/cromwell_application.conf \
+          --name CromwellContainer \
+          -e JAVA_OPTS="-Dconfig.file=/cromwell_application.conf" \
+          -d broadinstitute/cromwell:${CROMWELL_TAG} server
+
+          # Give cromwell server time to setup
+          sleep 1m
+
+      - name: Install Tox
+        run: pip install tox
+
+      - name: Run Tox integration tests
+        # Run tox using the version of Python in `PATH`
+        run: tox -e integration
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -1,4 +1,4 @@
-name: CI
+name: Unit Tests
 
 # Controls when the action will run.
 # Triggers the workflow on push or pull request events
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [ 3.6, 3.7, 3.8 ]
+        python: [ 3.7, 3.8, 3.9 ]
 
 
     # Steps represent a sequence of tasks that will be executed as part of the job
diff --git a/README.md b/README.md
@@ -93,7 +93,8 @@ Current version: 2.0.0
  It will copy your wdl and json inputs into the folder for reproducibility.  
  * It keeps track of your most recently submitted jobs by storing their ids in `./cromshell/`  
  You may omit the job ID of the last job submitted when running commands, or use negative numbers to reference previous jobs, e.g. "-1" will track the last job, "-2" will track the one before that, and so on.
- * You can override the default cromwell server by setting the environmental variable `CROMWELL_URL` to the appropriate URL.
+ * You can override the default cromwell server by setting the argument `--cromwell_url` to the appropriate URL.
+ * You can override the default cromshell configuration folder by setting the environmental variable `CROMSHELL_DIR` to the appropriate directory.
  * Most commands takes multiple workflow-ids, which you *can specify both in relative and absolute ID value* (i.e. `./cromwell status -1 -2 -3 c2db2989-2e09-4f2c-8a7f-c3733ae5ba7b`). 
 
 ## Installation
diff --git a/setup.py b/setup.py
@@ -17,14 +17,14 @@
     install_requires="""
     dataclasses
     termcolor
-    click
+    click>=8.0.0
     requests
     pygments
     """.split(
         "\n"
     ),
     tests_require=["coverage", "pytest"],
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     packages=find_packages("src"),
     package_dir={"": "src"},
     classifiers=[
@@ -33,9 +33,9 @@
         "License :: OSI Approved :: BSD 3-Clause",
         "Natural Language :: English",
         "Operating System :: OS Independent",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
     ],
     entry_points={"console_scripts": ["cromshell=cromshell.__main__:main_entry"]},
diff --git a/src/cromshell/__main__.py b/src/cromshell/__main__.py
@@ -73,6 +73,14 @@ def main_entry(
     requests_timeout,
     requests_skip_certs,
 ):
+    """
+    Cromshell is a script for submitting workflows to a
+    cromwell server and monitoring / querying their results.\n
+    Notes:\n
+        - A hidden folder will be created on initial run. The hidden folder
+    (.../.cromshell) will be placed in users home directory but can be overridden
+    by setting environment variable 'CROMSHELL_CONFIG'.
+    """
     # Set up our log verbosity
     from . import log  # pylint: disable=C0415
 
diff --git a/src/cromshell/log.py b/src/cromshell/log.py
@@ -33,7 +33,7 @@ def configure_logging(verbosity):
     # Set logging level:
     log_level = logging.INFO
     if verbosity:
-        log_level = verbosity
+        log_level = int(verbosity)
 
     # Create our basic config:
     logging.basicConfig(level=log_level, format=format_string)
diff --git a/src/cromshell/submit/command.py b/src/cromshell/submit/command.py
@@ -105,8 +105,9 @@ def main(config, wdl, wdl_json, options_json, dependencies_zip):
     # TODO: Refactor these file manipulations into its own "cleanup" function?
     # If we get here, we successfully submitted the job and should track it locally:
     # 1. Create a directory to hold function input files, using server name
+    server_folder_name = config.get_local_folder_name()
     run_directory = Path(config.config_dir).joinpath(
-        config.local_folder_name, workflow_status["id"]
+        server_folder_name, workflow_status["id"]
     )
     io_utils.create_directory(run_directory)
 
diff --git a/src/cromshell/utilities/cromshellconfig.py b/src/cromshell/utilities/cromshellconfig.py
@@ -27,12 +27,11 @@
 # Defaults for variables will be set after functions have been defined
 config_dir = None
 SUBMISSION_FILE_NAME = "all.workflow.database.tsv"
-SUBMISSION_FILE_HEADER = "DATE\tCROMWELL_SERVER\tRUN_ID\tWDL_NAME\tSTATUS\tALIAS"
+SUBMISSION_FILE_HEADER = "DATE\tCROMWELL_SERVER\tRUN_ID\tWDL_NAME\tSTATUS\tALIAS\n"
 CROMSHELL_CONFIG_FILE_NAME = "cromshell_config.json"
 submission_file_path = None
 cromshell_config_options = None
 cromwell_server = None
-local_folder_name = None
 # Request defaults
 requests_connect_timeout = 5
 requests_verify_certs = True
@@ -125,8 +124,20 @@ def resolve_cromwell_config_server_address(server_user=None, workflow_id=None):
 def __get_config_dir():
     """Get Path To Cromshell Hidden Directory"""
 
-    config_path = os.path.join(Path.home(), ".cromshell")
-    Path.mkdir(Path(config_path), exist_ok=True)
+    # If env CROMSHELL_CONFIG set then use for cromshell hidden dir else use home dir.
+    if os.environ.get("CROMSHELL_CONFIG"):
+        LOGGER.info(
+            "Detected 'CROMSHELL_CONFIG' in environment, using {config_path} as "
+            "cromshell hidden directory."
+        )
+        config_path = os.path.join(os.environ.get("CROMSHELL_CONFIG"), ".cromshell")
+
+    else:
+        config_path = os.path.join(Path.home(), ".cromshell")
+
+    Path.mkdir(Path(config_path), exist_ok=True, parents=True)
+    LOGGER.info(f"Cromshell config directory set to {config_path}.")
+
     return config_path
 
 
@@ -181,6 +192,11 @@ def get_cromwell_api():
     return f"{cromwell_server}{API_STRING}"
 
 
+def get_local_folder_name():
+    """Return a string combining the cromwell server without http/https"""
+    return cromwell_server.replace("https://", "").replace("http://", "")
+
+
 def resolve_requests_connect_timeout(timeout_cli: int):
     """Override the default request timeout duration.
 
@@ -220,4 +236,3 @@ def resolve_requests_connect_timeout(timeout_cli: int):
     config_dir, CROMSHELL_CONFIG_FILE_NAME, CROMSHELL_CONFIG_OPTIONS_TEMPLATE
 )
 cromwell_server = __get_cromwell_server(cromshell_config_options)
-local_folder_name = cromwell_server.replace("https://", "").replace("http://", "")
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -1,3 +1,9 @@
 pytest
 pytest-cov
+pytest-dependency
 coverage >= 4.5
+click>=8.0.0
+requests
+termcolor
+requests
+pygments
diff --git a/tests/integration/test_placeholder.py b/tests/integration/test_placeholder.py
diff --git a/tests/integration/test_submit.py b/tests/integration/test_submit.py
@@ -0,0 +1,117 @@
+import json
+import os
+from pathlib import Path
+from traceback import print_exception
+
+import pytest
+from click.testing import CliRunner
+
+from cromshell.__main__ import main_entry as cromshell
+
+
+def run_cromshell_submit(
+    wdl: str, json_file: str, local_cromwell_url: str, exit_code: int
+):
+    """Run cromshell submit using CliRunner and assert job is successful"""
+
+    runner = CliRunner(mix_stderr=False)
+    # The absolute path will be passed to the invoke command because
+    # the test is being run in temp directory created by CliRunner.
+    absolute_wdl = str(Path(wdl).resolve())
+    absolute_json = str(Path(json_file).resolve())
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cromshell,
+            [
+                "--cromwell_url",
+                local_cromwell_url,
+                "--hide_logo",
+                "submit",
+                absolute_wdl,
+                absolute_json,
+            ],
+        )
+        assert result.exit_code == exit_code, (
+            f"\nSTDOUT:\n{result.stdout}"
+            f"\nSTDERR:\n{result.stderr}"
+            f"\nExceptions:\n{result.exception}"
+            f"\n{print_exception(*result.exc_info)}"
+        )
+        return result
+
+
+def workflow_id_in_txt_db(result, local_workflow_database_tsv: Path):
+    """Get workflow id and assert it is listed in the local workflow database tsv"""
+
+    stdout_substring_formatted = json.loads(result.stdout)
+    test_workflow_id = stdout_substring_formatted["id"]
+
+    # check if workflow id is in database tsv
+    with open(local_workflow_database_tsv, "r") as file1:
+
+        # read file content
+        readfile = file1.read()
+        # checking condition for string found or not
+        assert (
+            test_workflow_id in readfile
+        ), "Workflow ID was not found in /all.workflow.database.tsv"
+
+
+class TestSubmit:
+    @pytest.mark.dependency(name="test_submit")
+    @pytest.mark.parametrize(
+        "wdl, json_file, exit_code",
+        [
+            ("tests/workflows/helloWorld.wdl", "tests/workflows/helloWorld.json", 0),
+            ("tests/workflows/helloWorld.wdl", "tests/workflows/not_valid.json", 1),
+            ("tests/workflows/not_valid.wdl", "tests/workflows/helloWorld.json", 1),
+        ],
+    )
+    def test_submit(
+        self,
+        local_cromwell_url: str,
+        wdl: str,
+        json_file: str,
+        exit_code: int,
+        local_hidden_cromshell_folder: Path,
+        local_workflow_database_tsv: Path,
+    ):
+        # Run cromshell submit
+        result = run_cromshell_submit(
+            wdl=wdl,
+            json_file=json_file,
+            exit_code=exit_code,
+            local_cromwell_url=local_cromwell_url,
+        )
+
+        # If submission passed check workflow id in database tsv
+        if exit_code == 0:
+            workflow_id_in_txt_db(
+                result=result, local_workflow_database_tsv=local_workflow_database_tsv
+            )
+
+    @pytest.mark.dependency(depends=["test_submit"])
+    def test_submit_cromshell_folders_created(
+        self, local_server_folder, local_hidden_cromshell_folder
+    ):
+        # Created hidden cromshell directory
+        assert Path.exists(local_hidden_cromshell_folder)
+        # Created a directory to hold function input files, using server name
+        assert Path.exists(local_server_folder)
+
+    @pytest.fixture
+    def local_cromwell_url(self):
+        return "http://localhost:8000"
+
+    @pytest.fixture
+    def local_hidden_cromshell_folder(self):
+        return Path(os.environ.get("CROMSHELL_CONFIG")).joinpath(".cromshell")
+
+    @pytest.fixture
+    def local_workflow_database_tsv(self, local_hidden_cromshell_folder):
+        return Path(local_hidden_cromshell_folder).joinpath("all.workflow.database.tsv")
+
+    @pytest.fixture
+    def local_server_folder(self, local_hidden_cromshell_folder, local_cromwell_url):
+        stripped_cromwell_url = local_cromwell_url.replace("http://", "")
+        return Path(local_hidden_cromshell_folder).joinpath(stripped_cromwell_url)
diff --git a/tests/resources/README.md b/tests/resources/README.md
@@ -0,0 +1,6 @@
+# Testing Resources 
+
+### cromwell_application.conf
+The cromshell_application.conf file is used by cromwell during the intgratin testing.
+The file contains setting used by the cromwell server container to connect with an 
+existing mysql container. A description of the config file can be found [here](https://cromwell.readthedocs.io/en/stable/tutorials/ConfigurationFiles/).
diff --git a/tests/resources/cromwell_application.conf b/tests/resources/cromwell_application.conf
diff --git a/tests/unit/test_cromshellconfig.py b/tests/unit/test_cromshellconfig.py
diff --git a/tox.ini b/tox.ini
