RCAL-1074: Implement integration test

.github/workflows/run-unit-tests.yml

@@ -39,4 +39,4 @@ jobs:
       - name: Run unit tests with coverage
         run: |
           coverage run -m pytest
-          coverage report --fail-under=70
+          coverage report

docs/create_simulated_catalog.rst

@@ -54,4 +54,3 @@ The following example demonstrates how to programmatically create a simulated ca
     )
 
     print(f"Simulated catalog with {len(catalog)} sources created successfully")
-

docs/roman_catalog_handler.rst

@@ -29,7 +29,7 @@ The following example demonstrates how to use the `RomanCatalogHandler` class to
      reg_test_data = Path(test_bigdata)
 
      # Specify the catalog file
-     test_cat = reg_test_data / "r0000101001001001001_0001_wfi01_cat.asdf"
+     test_cat = reg_test_data / "r0000101001001001001_0001_wfi01_cat.parquet"
 
      # Create an instance of RomanCatalogHandler
      catalog_handler = RomanCatalogHandler(test_cat.as_posix())

pyproject.toml

@@ -4,62 +4,63 @@ description = "Library for photometric redshift determination using data from th
 readme = "README.md"
 requires-python = ">=3.10"
 authors = [
-    { name = "Roman calibration pipeline developers", email = "help@stsci.edu" },
+  { name = "Roman calibration pipeline developers", email = "help@stsci.edu" },
 ]
 classifiers = [
-    "Intended Audience :: Science/Research",
-    "Topic :: Scientific/Engineering :: Astronomy",
-    "Programming Language :: Python :: 3",
+  "Intended Audience :: Science/Research",
+  "Topic :: Scientific/Engineering :: Astronomy",
+  "Programming Language :: Python :: 3",
 ]
 dependencies = [
-    "asdf >=3.3.0",
-    "asdf-astropy >=0.5.0",
-    "astropy >=5.3.0",
-    "crds >=11.16.16",
-    "gwcs >=0.21.0",
-    "jsonschema >=4.8",
-    "numpy >=1.23",
-    "photutils >=1.13.0",
-    "pyparsing >=2.4.7",
-    "requests >=2.26",
-    "roman_datamodels>=0.22.0",
-    # "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
-    "scipy>=1.7.0",
-    "stcal>=1.10.0",
-    # "stcal @ git+https://github.com/spacetelescope/stcal.git@main",
-    "stpipe >=0.7.0",
-    # "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main",
-    "tweakwcs >=0.8.8",
-    "spherical-geometry >= 1.2.22",
-    "stsci.imagestats >= 1.6.3",
-    "drizzle >= 1.15.0",
-    "webbpsf >= 1.2.1",
-    # "pz-rail-lephare",
-    "pz-rail-lephare @ git+https://git@github.com/mairanteodoro/rail_lephare.git",
-    "lephare == 0.1.13",
+  "asdf >=3.3.0",
+  "asdf-astropy >=0.5.0",
+  "astropy >=5.3.0",
+  "crds >=11.16.16",
+  "gwcs >=0.21.0",
+  "jsonschema >=4.8",
+  "numpy >=1.23",
+  "photutils >=1.13.0",
+  "pyparsing >=2.4.7",
+  "requests >=2.26",
+  "roman_datamodels>=0.22.0",
+  # "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
+  "scipy>=1.7.0",
+  "stcal>=1.10.0",
+  # "stcal @ git+https://github.com/spacetelescope/stcal.git@main",
+  "stpipe >=0.7.0",
+  # "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main",
+  "tweakwcs >=0.8.8",
+  "spherical-geometry >= 1.2.22",
+  "stsci.imagestats >= 1.6.3",
+  "drizzle >= 1.15.0",
+  "webbpsf >= 1.2.1",
+  # "pz-rail-lephare",
+  "pz-rail-lephare @ git+https://git@github.com/mairanteodoro/rail_lephare.git",
+  "lephare == 0.1.13",
+  "pz-rail-base == 1.1.5",
 ]
 license-files = ["LICENSE"]
 dynamic = ["version"]
 
 [project.optional-dependencies]
 docs = [
-    "matplotlib",
-    "sphinx",
-    "sphinx-astropy",
-    "sphinx-automodapi",
-    "sphinx-rtd-theme",
-    "stsci-rtd-theme",
-    "sphinx-autobuild",
-    "tomli; python_version <=\"3.11\"",
-    "sphinx-argparse",
+  "matplotlib",
+  "sphinx",
+  "sphinx-astropy",
+  "sphinx-automodapi",
+  "sphinx-rtd-theme",
+  "stsci-rtd-theme",
+  "sphinx-autobuild",
+  "tomli; python_version <=\"3.11\"",
+  "sphinx-argparse",
 ]
 test = [
-    "ci-watson >=0.5.0",
-    "pytest >8.0.0",
-    "pytest-astropy >= 0.11.0",
-    "deepdiff",
-    "stpreview>=0.5.1",
-    "pytest-cov",
+  "ci-watson >=0.5.0",
+  "pytest >8.0.0",
+  "pytest-astropy >= 0.11.0",
+  "deepdiff",
+  "stpreview>=0.5.1",
+  "pytest-cov",
 ]
 dev = ["roman_photoz[docs,test]", "tox > 4", "pre-commit > 3"]
 sdp = ["stpreview>=0.5.1"]
@@ -83,14 +84,14 @@ zip-safe = false
 [tool.setuptools.package-data]
 # package_data values are glob patterns relative to each specific subpackage.
 "*" = [
-    "**/*.fits",
-    "**/*.txt",
-    "**/*.inc",
-    "**/*.cfg",
-    "**/*.csv",
-    "**/*.yaml",
-    "**/*.json",
-    "**/*.asdf",
+  "**/*.fits",
+  "**/*.txt",
+  "**/*.inc",
+  "**/*.cfg",
+  "**/*.csv",
+  "**/*.yaml",
+  "**/*.json",
+  "**/*.asdf",
 ]
 
 [tool.pytest.ini_options]
@@ -120,7 +121,7 @@ line-length = 88
 [tool.ruff.lint]
 exclude = ["jdocs", ".tox", ".eggs", "build"]
 ignore = [
-    "E741", # ambiguous variable name
+  "E741", # ambiguous variable name
 ]
 extend-select = ["NPY"]
 
@@ -132,16 +133,13 @@ archs = ["auto", "aarch64"]
 
 [tool.coverage.run]
 source = ["roman_photoz"]
-omit = [
-    "*/tests/*",
-    "*/docs/*",
-]
+omit = ["*/tests/*", "*/docs/*"]
 
 [tool.coverage.report]
 show_missing = true
 skip_covered = true
 precision = 2
-fail_under = 70
+# fail_under = 70
 
 [tool.coverage.html]
 directory = "htmlcov"

roman_photoz/create_simulated_catalog.py

@@ -7,9 +7,9 @@
 
 import lephare as lp
 import numpy as np
+from astropy.table import Table
 from numpy.lib import recfunctions as rfn
 from rail.core.stage import RailStage
-from roman_datamodels import datamodels as rdm
 
 from roman_photoz import create_roman_filters
 from roman_photoz.default_config_file import default_roman_config
@@ -25,7 +25,7 @@
     "LEPHAREWORK", (Path(LEPHAREDIR).parent / "work").as_posix()
 )
 CWD = os.getcwd()
-DEFAULT_OUTPUT_CATALOG_FILENAME = "roman_simulated_catalog.asdf"
+DEFAULT_OUTPUT_CATALOG_FILENAME = "roman_simulated_catalog.parquet"
 
 
 class SimulatedCatalog:
@@ -73,10 +73,10 @@ def __init__(self):
         self.roman_catalog_template = self.read_roman_template_catalog()
 
     def read_roman_template_catalog(self):
-        input_filename = "roman_catalog_template.asdf"
+        input_filename = "r00001_p_v01001001001001_270p65x49y70_f158_mbcat_cat.parquet"
         this_path = Path(__file__).resolve().parent
         input_path = (this_path / "data" / input_filename).as_posix()
-        return rdm.open(input_path)
+        return Table.read(input_path)
 
     def is_folder_not_empty(self, folder_path: str, partial_text: str) -> bool:
         """
@@ -166,6 +166,13 @@ def create_simulated_input_catalog(
     ):
         """
         Create a simulated input catalog from the simulated data.
+
+        + read the ROMAN_SIMULATED_MAGS.dat produced by LePhare's
+            `prepare` method in `create_simulated_input_catalog`. The
+            columns name are defined by LePhare.
+
+        + format the columns name to match Roman catalog's specifications
+
         """
         catalog_name = Path(
             LEPHAREWORK, "lib_mag", f"{self.simulated_data_filename}.dat"
@@ -184,7 +191,7 @@ def create_simulated_input_catalog(
         ]
 
         # we're matching the number of objects in the template
-        num_lines = len(self.roman_catalog_template.source_catalog)
+        num_lines = len(self.roman_catalog_template)
         random_lines = self.pick_random_lines(num_lines)
         catalog = random_lines[cols_to_keep]
 
@@ -227,67 +234,77 @@ def update_roman_catalog_template(self, catalog):
         """
         filter_list = (
             default_roman_config["FILTER_LIST"]
+            .lower()
             .replace("roman/roman_", "")
             .replace(".pb", "")
             .split(",")
         )
-        # in the asdf template file we only have the flux in
+        # TODO: in the template file we only have the flux in
         # the F158 filter so we're adding the other filters
+        # get a list of all the column names (parameters) in the template catalog
+        # and remove the filter name since it is a monochromatic catalog (only F158)
         roman_filter_params = [
-            x.replace("F158", "")
-            for x in self.roman_catalog_template.source_catalog.columns
-            if "F158" in x
+            x.lower().replace("f158", "")
+            for x in self.roman_catalog_template.dtype.names
+            if "f158" in x.lower()
         ]
 
         # # first, clear the template
         # self.roman_catalog_template.source_catalog.remove_rows(slice(None))
-        self.roman_catalog_template.source_catalog.add_column(catalog["id"], name="id")
-
+        # self.roman_catalog_template.source_catalog.add_column(
+        #     catalog["label"], name="label"
+        # )
         # then add the simulated data
         for filter_name in filter_list:
             for param in roman_filter_params:
-                new_column = f"{filter_name}{param}"
-                if new_column not in self.roman_catalog_template.source_catalog.columns:
-                    # add new column
-                    if "flux" in new_column:
+                # the new column name has to match the on in the template catalog,
+                # so we need to put the filter name back in the parameter
+                col_name = (
+                    f"{param}{filter_name}"
+                    if "__" not in param
+                    else f"_{filter_name}_".join(param.split("__"))
+                )
+                if col_name not in self.roman_catalog_template.dtype.names:
+                    # add new column (data for a missing filter)
+                    # N.B.: we only need to add photometric data for the missing filters
+                    # so we skip all the other parameters that are not fluxes
+                    if "flux" in col_name:
                         # add flux and error columns for each filter
+                        # 1 - grab simulated flux or flux error from LePhare
                         simulated_colname = (
-                            f"magnitude_{filter_name}_err"
-                            if "err" in new_column
-                            else f"magnitude_{filter_name}"
+                            f"magnitude_{filter_name.capitalize()}_err"
+                            if "err" in col_name
+                            else f"magnitude_{filter_name.capitalize()}"
                         )
-                        self.roman_catalog_template.source_catalog.add_column(
-                            catalog[simulated_colname], name=new_column
+                        # 2 - add the new column to the template catalog
+                        # (with the data corresponding to the missing filter)
+                        self.roman_catalog_template.add_column(
+                            catalog[simulated_colname], name=col_name
                         )
                     else:
-                        # copy parameter from F158
-                        simulated_colname = new_column
-                        self.roman_catalog_template.source_catalog.add_column(
-                            self.roman_catalog_template.source_catalog[f"F158{param}"],
-                            name=new_column,
+                        # for non-photometric data, just copy parameter from F158
+                        self.roman_catalog_template.add_column(
+                            self.roman_catalog_template[
+                                f"{col_name.replace(filter_name, 'f158')}"
+                            ],
+                            name=col_name,
                         )
 
                 else:
                     # replace column data
-                    if "flux" in new_column:
+                    if "flux" in col_name:
                         simulated_colname = (
-                            f"magnitude_{filter_name}_err"
-                            if "err" in new_column
-                            else f"magnitude_{filter_name}"
-                        )
-                        self.roman_catalog_template.source_catalog[new_column] = (
-                            catalog[simulated_colname]
+                            f"magnitude_{filter_name.capitalize()}_err"
+                            if "err" in col_name
+                            else f"magnitude_{filter_name.capitalize()}"
                         )
+                        self.roman_catalog_template[col_name] = catalog[
+                            simulated_colname
+                        ]
 
-        self.roman_catalog_template.source_catalog.add_column(
-            catalog["context"], name="context"
-        )
-        self.roman_catalog_template.source_catalog.add_column(
-            catalog["zspec"], name="zspec"
-        )
-        self.roman_catalog_template.source_catalog.add_column(
-            catalog["z_true"], name="string_data"
-        )
+        self.roman_catalog_template.add_column(catalog["context"], name="context")
+        self.roman_catalog_template.add_column(catalog["zspec"], name="zspec")
+        self.roman_catalog_template.add_column(catalog["z_true"], name="string_data")
 
     def save_catalog(
         self,
@@ -302,10 +319,8 @@ def save_catalog(
         output_filename : str, optional
             The filename to save the catalog to.
         """
-        output_path = Path(output_path).as_posix()
-        output_filename = output_filename
-        logger.info(f"Saving catalog to {output_path}/{output_filename}...")
-        self.roman_catalog_template.save(output_filename, dir_path=output_path)
+        logger.info(f"Saving catalog to {Path(output_path)}/{output_filename}...")
+        self.roman_catalog_template.write(Path(output_path, output_filename))
         logger.info("Catalog saved successfully")
 
     def add_ids(self, catalog):
@@ -323,10 +338,12 @@ def add_ids(self, catalog):
             The catalog data with an ID column added.
         """
         ids = np.arange(1, len(catalog) + 1)
-        catalog = rfn.append_fields(catalog, "id", ids, usemask=False)
+        catalog = rfn.append_fields(catalog, "label", ids, usemask=False)
 
-        new_dtype = [("id", catalog["id"].dtype)] + [
-            (name, catalog[name].dtype) for name in catalog.dtype.names if name != "id"
+        new_dtype = [("label", catalog["label"].dtype)] + [
+            (name, catalog[name].dtype)
+            for name in catalog.dtype.names
+            if name != "label"
         ]
         new_catalog = np.empty(catalog.shape, dtype=new_dtype)
         for name in new_catalog.dtype.names:

roman_photoz/default_config_file.py

@@ -20,7 +20,7 @@
     "AGE_RANGE": "0.,15.e9",
     "AUTO_ADAPT": "YES",
     "CAT_FMT": "MEME",
-    "CAT_IN": "roman_simulated_catalog.asdf",
+    "CAT_IN": "roman_simulated_catalog.parquet",
     "CAT_LINES": "-99,-99",
     "CAT_MAG": "VEGA",
     "CAT_OUT": "roman_simulated_catalog.out",