Add coverage report for infrastructure tests

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+omit =
+    *fv3core/stencils/*
+    */setup.py
+    */tests/*

.gitignore

@@ -1,3 +1,5 @@
+coverage_html_report
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.jenkins/actions/run_regression_tests.sh

@@ -10,8 +10,12 @@ make get_test_data
 
 if [ ${python_env} == "virtualenv" ]; then
     export TEST_ARGS="${TEST_ARGS} --junitxml=${jenkins_dir}/${XML_REPORT}"
-    CONTAINER_CMD="srun" make tests savepoint_tests
+    export CONTAINER_CMD="srun"
+    COVERAGE=y make tests
+    make savepoint_tests
 else
     export TEST_ARGS="${TEST_ARGS} --junitxml=/.jenkins/${XML_REPORT}"
-    VOLUMES="-v ${pwd}/.jenkins:/.jenkins" make tests savepoint_tests
+    export VOLUMES="-v ${pwd}/.jenkins:/.jenkins"
+    COVERAGE=y make tests
+    make savepoint_tests
 fi

Makefile

@@ -17,6 +17,7 @@ RUN_FLAGS ?=--rm
 TEST_ARGS ?=
 TEST_DATA_HOST ?=$(CWD)/test_data/$(EXPERIMENT)
 FV3UTIL_DIR=$(CWD)/external/fv3gfs-util
+COVERAGE ?=n
 
 FV3=fv3core
 FV3_PATH ?=/$(FV3)
@@ -33,13 +34,17 @@ DATA_BUCKET= $(REGRESSION_DATA_STORAGE_BUCKET)/$(FORTRAN_SERIALIZED_DATA_VERSION
 
 TEST_TYPE=$(word 3, $(subst _, ,$(EXPERIMENT)))
 THRESH_ARGS=--threshold_overrides_file=$(FV3_PATH)/tests/savepoint/translate/overrides/$(TEST_TYPE).yaml
-PYTEST_MAIN=pytest $(TEST_ARGS) $(FV3_PATH)/tests/main
-PYTEST_SEQUENTIAL=pytest --data_path=$(TEST_DATA_RUN_LOC) $(TEST_ARGS) $(THRESH_ARGS) $(FV3_PATH)/tests/savepoint
 # we can't rule out a deadlock if one test fails, so we must set maxfail=1 for parallel tests
-PYTEST_PARALLEL=$(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(TEST_DATA_RUN_LOC) $(TEST_ARGS) $(THRESH_ARGS) -m parallel $(FV3_PATH)/tests/savepoint
 ifeq ($(DEV),y)
 	VOLUMES += -v $(CWD)/$(FV3):/$(FV3)/$(FV3) -v $(CWD)/tests:/$(FV3)/tests -v $(FV3UTIL_DIR):/usr/src/fv3gfs-util
 endif
+ifeq ($(COVERAGE),y)
+	VOLUMES += -v $(CWD)/coverage_html_report:/coverage_html_report
+	TEST_ARGS += --cov=fv3core --cov-report html:coverage_html_report --cov-report term --cov-config=/fv3core/.coveragerc
+endif
+PYTEST_MAIN=pytest $(TEST_ARGS) $(FV3_PATH)/tests/main
+PYTEST_SEQUENTIAL=pytest --data_path=$(TEST_DATA_RUN_LOC) $(TEST_ARGS) $(THRESH_ARGS) $(FV3_PATH)/tests/savepoint
+PYTEST_PARALLEL=$(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(TEST_DATA_RUN_LOC) $(TEST_ARGS) $(THRESH_ARGS) -m parallel $(FV3_PATH)/tests/savepoint
 CONTAINER_CMD?=$(CONTAINER_ENGINE) run $(RUN_FLAGS) $(VOLUMES) $(CUDA_FLAGS) $(FV3CORE_IMAGE)
 
 clean:
@@ -55,8 +60,8 @@ update_submodules_base:
 
 update_submodules_venv: update_submodules_base
 	if [ ! -f $(CWD)/external/daint_venv/install.sh  ]; then \
-                git submodule update --init external/daint_venv; \
-        fi
+        git submodule update --init external/daint_venv; \
+    fi
 
 constraints.txt: requirements.txt requirements/requirements_wrapper.txt requirements/requirements_lint.txt
 	pip-compile $^ --output-file constraints.txt
@@ -149,6 +154,7 @@ dev:
 		--network host \
 		-v $(TEST_DATA_HOST):$(TEST_DATA_RUN_LOC) \
 		-v $(CWD):/port_dev \
+		$(VOLUMES) \
 		$(FV3CORE_IMAGE) bash
 
 dev_wrapper:

docker/Dockerfile

@@ -45,7 +45,7 @@ ENV FV3_STENCIL_REBUILD_FLAG=False
 COPY external/fv3gfs-util/ /external/fv3gfs-util/
 COPY fv3core /fv3core/fv3core
 COPY tests /fv3core/tests
-COPY setup.py setup.cfg README.md /fv3core/
+COPY setup.py setup.cfg README.md .coveragerc /fv3core/
 COPY docker/entrypoint.sh /entrypoint.sh
 
 # Docker hard limits shared memory usage. MPICH for oversubscribed situation