rcal-627 Add metrics logger to regtests (#831)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: ddavis-stsci

CHANGES.rst

@@ -46,6 +46,8 @@ refpix
 general
 -------
 
+- Add metrics_logger to the regression tests [#831]
+
 - Update pipeline logic for saturation checks [#824]
 
 - Update the pipeline code to process all the uncal files in an association [#802]

pyproject.toml

@@ -49,6 +49,7 @@ test = [
     'ci-watson >=0.5.0',
     'pytest >=4.6.0',
     'pytest-astropy',
+    'metrics_logger >= 0.1.0',
 ]
 
 [project.urls]

requirements-dev.txt

@@ -12,3 +12,4 @@ scipy>=0.0.dev0
 git+https://github.com/spacetelescope/stpipe
 git+https://github.com/spacetelescope/stcal
 git+https://github.com/spacetelescope/tweakwcs
+#git+https://github.com/spacetelescope/metrics_logger

romancal/regtest/test_tweakreg.py

@@ -1,6 +1,7 @@
 from io import StringIO
 
 import pytest
+from metrics_logger.decorators import metrics_logger
 from roman_datamodels import datamodels as rdm
 
 from romancal.stpipe import RomanStep
@@ -44,6 +45,7 @@ def create_asn_file():
     return asn_file_path
 
 
+@metrics_logger("DMS280")
 @pytest.mark.bigdata
 def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path):
     # N.B.: the data were created using WFIsim and processed through

romancal/regtest/test_wfi_dq_init.py

@@ -3,13 +3,15 @@
 
 import pytest
 import roman_datamodels as rdm
+from metrics_logger.decorators import metrics_logger
 
 from romancal.step import DQInitStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
+@metrics_logger("DMS25")
 @pytest.mark.bigdata
 def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     """DMS25 Test: Testing retrieval of best ref file for image data,
@@ -64,6 +66,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     assert compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None
 
 
+@metrics_logger("DMS26")
 @pytest.mark.bigdata
 def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
     """DMS26 Test: Testing retrieval of best ref file for grism data,

romancal/regtest/test_wfi_flat_field.py

@@ -3,6 +3,7 @@
 import pytest
 import roman_datamodels as rdm
 from crds.core.exceptions import CrdsLookupError
+from metrics_logger.decorators import metrics_logger
 
 from romancal.step import FlatFieldStep
 from romancal.stpipe import RomanStep
@@ -66,6 +67,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
     assert compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None
 
 
+@metrics_logger("DMS79")
 @pytest.mark.bigdata
 @pytest.mark.soctests
 def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):

romancal/regtest/test_wfi_photom.py

@@ -4,13 +4,15 @@
 import pytest
 import roman_datamodels as rdm
 from astropy import units as u
+from metrics_logger.decorators import metrics_logger
 
 from romancal.step import PhotomStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
+@metrics_logger("DMS140")
 @pytest.mark.bigdata
 def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     """DMS140 Test: Testing application of photometric correction using

romancal/regtest/test_wfi_pipeline.py

@@ -5,6 +5,7 @@
 import pytest
 import roman_datamodels as rdm
 from gwcs.wcstools import grid_from_bounding_box
+from metrics_logger.decorators import metrics_logger
 from numpy.testing import assert_allclose
 
 from romancal.assign_wcs.assign_wcs_step import AssignWcsStep
@@ -22,6 +23,7 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
+@metrics_logger("DMS86", "DMS87", "DMS88")
 def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
     """Tests for flat field imaging processing requirements DMS86 & DMS 87"""
     input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
@@ -254,6 +256,7 @@ def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
+@metrics_logger("DMS90", "DMS91", "DMS9")
 def test_level2_grism_processing_pipeline(rtdata, ignore_asdf_paths):
     """Tests for flat field grism processing requirements DMS90 & DMS 91"""
     input_data = "r0000201001001001002_01101_0001_WFI01_uncal.asdf"

tox.ini

@@ -54,7 +54,8 @@ pass_env =
     CODECOV_*
     DD_*
 set_env =
-    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simpl
+
 extras =
     test
     alldeps: all
@@ -73,7 +74,7 @@ commands =
     cov: --cov --cov-config pyproject.toml --cov-report term-missing --cov-report xml \
     warnings: -W error \
     regtests: --bigdata --slow --basetemp={homedir}/test_outputs \
-    xdist: -n auto \
+    xdist: -n 0 \
     pyargs: {toxinidir}/docs --pyargs {posargs:romancal} \
     ddtrace: --ddtrace \
     {posargs}