RAD-151: L2 and L3 catalog and segmentation map schemas (#393)

* adding source catalog schema

* fix changelog

* adding segmentation map schema

* Update src/rad/resources/schemas/source_catalog-1.0.0.yaml

Co-authored-by: William Jamieson <wjamieson@stsci.edu>

* add meta to source catalog

* Update CHANGES.rst

Co-authored-by: Nadia Dencheva <nadia.astropy@gmail.com>

* Update src/rad/resources/schemas/segmentation_map-1.0.0.yaml

Co-authored-by: Nadia Dencheva <nadia.astropy@gmail.com>

* Updates from Nadia's review comments

* fixing meta and binary block order

* tweaking metadata requirements

* this will be L3 schema

* add mosaic_source_catalog and mosaic_segmentation_map

* L2 catalogs and segmentation maps

* add optical_element

* add change log entry

* address comments

* address comments

* Add database destination tables for mosaic_source_catalog

* add archive destinations for L2 catalogs

* add segment destination

* remove SegmentationMap table from exposure schema

* add photometry to L2 catalogs

---------

Co-authored-by: Brett M. Morris <bmmorris@stsci.edu>
Co-authored-by: Brett M. Morris <morrisbrettm@gmail.com>
Co-authored-by: William Jamieson <wjamieson@stsci.edu>

Author: 3 people

CHANGES.rst

@@ -1,4 +1,4 @@
-0.19.1 (unreleased)
+0.20.0 (unreleased)
 -------------------
 
 - Add new schemas to documentation. [#386]
@@ -26,6 +26,7 @@
 
 - Create the flux step schema. [#395]
 
+- Add source_catalog and segmentation_map schemas for Level 2 and Level 3 files. [#393]
 
 0.19.0 (2024-02-09)
 -------------------

docs/creating.rst

@@ -386,6 +386,15 @@ actually functions; meaning that for ASDF 3.0+ to correctly handle the schema
 without issues, the ``allOf`` combiner is necessary, see
 `PR 222 <https://github.com/spacetelescope/rad/pull/222>`_ for more details.
 
+Testing Schemas
+---------------
+
+Once you created a schema, run the tests in the ``rad`` package before proceeding
+to write the model.
+
+.. note::
+     The schemas need to be committed to the working repository and the ``rad``
+     package needs to be installed before running the tests.
 
 Creating a Data Model
 ---------------------

src/rad/resources/manifests/datamodels-1.0.yaml

@@ -268,6 +268,26 @@ tags:
   title: Calibration reference file names.
   description: |-
     Calibration reference file names.
+- tag_uri: asdf://stsci.edu/datamodels/roman/tags/source_catalog-1.0.0
+  schema_uri: asdf://stsci.edu/datamodels/roman/schemas/source_catalog-1.0.0
+  title: Source catalog
+  description: |-
+    Photometry and astrometry computed by the Source Catalog Step
+- tag_uri: asdf://stsci.edu/datamodels/roman/tags/segmentation_map-1.0.0
+  schema_uri: asdf://stsci.edu/datamodels/roman/schemas/segmentation_map-1.0.0
+  title: Segmentation map
+  description: |-
+    Segmentation map computed by the Source Catalog Step
+- tag_uri: asdf://stsci.edu/datamodels/roman/tags/mosaic_source_catalog-1.0.0
+  schema_uri: asdf://stsci.edu/datamodels/roman/schemas/mosaic_source_catalog-1.0.0
+  title: Mosaic source catalog
+  description: |-
+    Photometry and astrometry computed by the Source Catalog Step
+- tag_uri: asdf://stsci.edu/datamodels/roman/tags/mosaic_segmentation_map-1.0.0
+  schema_uri: asdf://stsci.edu/datamodels/roman/schemas/mosaic_segmentation_map-1.0.0
+  title: Mosaic segmentation map
+  description: |-
+    Segmentation map computed by the Source Catalog Step
 # Tagged Scalars
 - tag_uri: asdf://stsci.edu/datamodels/roman/tags/calibration_software_version-1.0.0
   schema_uri: asdf://stsci.edu/datamodels/roman/schemas/tagged_scalars/calibration_software_version-1.0.0

src/rad/resources/schemas/exposure-1.0.0.yaml

@@ -22,7 +22,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: int
-          destination: [WFIExposure.exposure_id, GuideWindow.exposure_id]
+          destination: [WFIExposure.exposure_id, GuideWindow.exposure_id,
+                        SourceCatalog.exposure_id]
       mid_time:
         title: Exposure Mid Time (UTC)
         description: |
@@ -34,7 +35,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: datetime2
-          destination: [WFIExposure.exposure_mid_time, GuideWindow.exposure_mid_time]
+          destination: [WFIExposure.exposure_mid_time, GuideWindow.exposure_mid_time,
+                        SourceCatalog.exposure_mid_time]
       end_time:
         title: Exposure End Time (UTC)
         description: |
@@ -46,7 +48,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: datetime2
-          destination: [WFIExposure.exposure_end_time, GuideWindow.exposure_end_time]
+          destination: [WFIExposure.exposure_end_time, GuideWindow.exposure_end_time,
+                        SourceCatalog.exposure_end_time]
       start_time_mjd:
         title: MJD Start Time (d)
         description: |
@@ -59,7 +62,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_start_time_mjd, GuideWindow.exposure_start_time_mjd]
+          destination: [WFIExposure.exposure_start_time_mjd, GuideWindow.exposure_start_time_mjd,
+                        SourceCatalog.exposure_start_time_mjd]
       mid_time_mjd:
         title: MJD Mid Time (d)
         description: |
@@ -72,7 +76,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_mid_time_mjd, GuideWindow.exposure_mid_time_mjd]
+          destination: [WFIExposure.exposure_mid_time_mjd, GuideWindow.exposure_mid_time_mjd,
+                        SourceCatalog.exposure_mid_time_mjd]
       end_time_mjd:
         title: MJD End Time (d)
         description: |
@@ -85,7 +90,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_end_time_mjd, GuideWindow.exposure_end_time_mjd]
+          destination: [WFIExposure.exposure_end_time_mjd, GuideWindow.exposure_end_time_mjd,
+                        SourceCatalog.exposure_end_time_mjd]
       start_time_tdb:
         title: TDB Start Time (d)
         description: |
@@ -98,7 +104,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_start_time_tdb, GuideWindow.exposure_start_time_tdb]
+          destination: [WFIExposure.exposure_start_time_tdb, GuideWindow.exposure_start_time_tdb,
+                        SourceCatalog.exposure_start_time_tdb]
       mid_time_tdb:
         title: TDB Mid Time (d)
         description: |
@@ -111,7 +118,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_mid_time_tdb, GuideWindow.exposure_mid_time_tdb]
+          destination: [WFIExposure.exposure_mid_time_tdb, GuideWindow.exposure_mid_time_tdb,
+                        SourceCatalog.exposure_mid_time_tdb]
       end_time_tdb:
         title: TDB End Time (d)
         description: |
@@ -124,7 +132,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_end_time_tdb, GuideWindow.exposure_end_time_tdb]
+          destination: [WFIExposure.exposure_end_time_tdb, GuideWindow.exposure_end_time_tdb,
+                        SourceCatalog.exposure_end_time_tdb]
       sca_number:
         title: SCA Number
         description: |
@@ -137,7 +146,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: int
-          destination: [WFIExposure.exposure_sca_number, GuideWindow.exposure_sca_number]
+          destination: [WFIExposure.exposure_sca_number, GuideWindow.exposure_sca_number,
+                        SourceCatalog.exposure_sca_number]
       gain_factor:
         title: Gain Factor
         type: number
@@ -147,7 +157,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_gain_factor, GuideWindow.exposure_gain_factor]
+          destination: [WFIExposure.exposure_gain_factor, GuideWindow.exposure_gain_factor,
+                        SourceCatalog.exposure_gain_factor]
       integration_time:
         title: Effective Integration Time (s)
         description:
@@ -159,7 +170,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_integration_time, GuideWindow.exposure_integration_time]
+          destination: [WFIExposure.exposure_integration_time, GuideWindow.exposure_integration_time,
+                        SourceCatalog.exposure_integration_time]
       elapsed_exposure_time:
         title: Elapsed Exposure Time (s)
         description: |
@@ -172,7 +184,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.elapsed_exposure_time, GuideWindow.elapsed_exposure_time]
+          destination: [WFIExposure.elapsed_exposure_time, GuideWindow.elapsed_exposure_time,
+                        SourceCatalog.elapsed_exposure_time]
       effective_exposure_time:
         title: Effective Exposure Time (s)
         description: |
@@ -185,7 +198,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.effective_exposure_time, GuideWindow.effective_exposure_time]
+          destination: [WFIExposure.effective_exposure_time, GuideWindow.effective_exposure_time,
+                        SourceCatalog.effective_exposure_time]
       duration:
         title: Exposure Duration (s)
         description: |
@@ -198,7 +212,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: float
-          destination: [WFIExposure.exposure_duration, GuideWindow.exposure_duration]
+          destination: [WFIExposure.exposure_duration, GuideWindow.exposure_duration,
+                        SourceCatalog.exposure_duration]
       level0_compressed:
         title: Level 0 Compression
         description: |
@@ -211,7 +226,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: nchar(1)
-          destination: [WFIExposure.exposure_level0_compressed, GuideWindow.exposure_level0_compressed]
+          destination: [WFIExposure.exposure_level0_compressed, GuideWindow.exposure_level0_compressed,
+                        SourceCatalog.exposure_level0_compressed]
       truncated:
         title: Truncated MA Table
         description: |
@@ -223,7 +239,8 @@ allOf:
             origin: TBD
         archive_catalog:
           datatype: nchar(1)
-          destination: [WFIExposure.exposure_truncated]
+          destination: [WFIExposure.exposure_truncated,
+                        SourceCatalog.exposure_truncated]
     required: [id,
               mid_time, end_time,
               start_time_mjd, mid_time_mjd, end_time_mjd,

src/rad/resources/schemas/mosaic_basic-1.0.0.yaml

@@ -16,7 +16,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: float
-      destination: [WFIMosaic.time_first_mjd]
+      destination: [WFIMosaic.time_first_mjd, SourceCatalog.time_first_mjd, SegmentationMap.time_first_mjd]
   time_last_mjd:
     title: Latest component image end time in the mosaic in MJD
     type: number
@@ -26,7 +26,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: float
-      destination: [WFIMosaic.time_last_mjd]
+      destination: [WFIMosaic.time_last_mjd, SourceCatalog.time_last_mjd, SegmentationMap.time_last_mjd]
   time_mean_mjd:
     title: Mean of mid-times of component images in MJD
     type: number
@@ -36,7 +36,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: float
-      destination: [WFIMosaic.time_mean_mjd]
+      destination: [WFIMosaic.time_mean_mjd, SourceCatalog.time_mean_mjd, SegmentationMap.time_mean_mjd]
   max_exposure_time:
     title: Maximum component image exposure time in MJD
     type: number
@@ -46,7 +46,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: float
-      destination: [WFIMosaic.max_exposure_time]
+      destination: [WFIMosaic.max_exposure_time, SourceCatalog.max_exposure_time, SegmentationMap.max_exposure_time]
   mean_exposure_time:
     title: Mean of component image exposure times in MJD
     type: number
@@ -56,7 +56,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: float
-      destination: [WFIMosaic.mean_exposure_time]
+      destination: [WFIMosaic.mean_exposure_time, SourceCatalog.mean_exposure_time, SegmentationMap.mean_exposure_time]
   visit:
     title: Visit number within the observation, defined range of
            values is 1..999; included in obs_id and visit_id as 'VVV'.
@@ -67,7 +67,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: smallint
-      destination: [WFIMosaic.visit]
+      destination: [WFIMosaic.visit, SourceCatalog.visit, SegmentationMap.visit]
   segment:
     title: Segment Number within pass, defined range is 1..999;
            included in obs_id and visit_id as 'SSS'.
@@ -78,7 +78,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: smallint
-      destination: [WFIMosaic.segment]
+      destination: [WFIMosaic.segment, SourceCatalog.segment, SegmentationMap.segment]
   pass:
     title: Pass number within execution plan, defined range is 1..999;
            included in obs_id and visit_id as 'AA'.
@@ -89,7 +89,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: smallint
-      destination: [WFIMosaic.pass]
+      destination: [WFIMosaic.pass, SourceCatalog.pass, SegmentationMap.pass]
   program:
     title: Program number, defined range is 1..18445;
            included in obs_id and visit_id as 'PPPPP'.
@@ -100,13 +100,13 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: int
-      destination: [WFIMosaic.program]
+      destination: [WFIMosaic.program, SourceCatalog.program, SegmentationMap.program]
   survey:
     title: Observation Survey
     type: string
     archive_catalog:
       datatype: nvarchar(15)
-      destination: [WFIMosaic.survey]
+      destination: [WFIMosaic.survey, SourceCatalog.survey, SegmentationMap.survey]
   optical_element:
     $ref: wfi_optical_element-1.0.0
     sdf:
@@ -115,7 +115,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: nvarchar(20)
-      destination: [WFIMosaic.optical_element]
+      destination: [WFIMosaic.optical_element, SourceCatalog.optical_element, SegmentationMap.optical_element]
   instrument:
     title: Instrument used to acquire the data
     type: string
@@ -126,7 +126,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: nvarchar(5)
-      destination: [WFIMosaic.instrument_name]
+      destination: [WFIMosaic.instrument_name, SourceCatalog.instrument_name, SegmentationMap.instrument_name]
   location_name:
     title: Name of the skycell containing the mosaic
     type: string
@@ -136,7 +136,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: nvarchar(25)
-      destination: [WFIMosaic.location_name]
+      destination: [WFIMosaic.location_name, SourceCatalog.location_name, SegmentationMap.location_name]
   product_type:
     title: Association product type
     type: string
@@ -146,7 +146,7 @@ properties:
         origin: TBD
     archive_catalog:
       datatype: nvarchar(25)
-      destination: [WFIMosaic.product_type]
+      destination: [WFIMosaic.product_type, SourceCatalog.product_type, SegmentationMap.product_type]
 propertyOrder: [ time_first_mjd, time_last_mjd, time_mean_mjd, max_exposure_time,
                  mean_exposure_time, visit, segment, pass, program,
                  survey, optical_element, instrument, location_name, product_type]

src/rad/resources/schemas/mosaic_segmentation_map-1.0.0.yaml

@@ -0,0 +1,36 @@
+%YAML 1.1
+---
+$schema: asdf://stsci.edu/datamodels/roman/schemas/rad_schema-1.0.0
+id: asdf://stsci.edu/datamodels/roman/schemas/mosaic_segmentation_map-1.0.0
+
+title: Segmentation map generated from a Level 3 file by the Source Catalog Step.
+
+datamodel_name: MosaicSegmentationMapModel
+
+archive_meta: None
+type: object
+properties:
+  meta:
+    allOf:
+      - $ref: basic-1.0.0
+      - type: object
+        properties:
+          basic:
+            tag: asdf://stsci.edu/datamodels/roman/tags/mosaic_basic-1.0.0
+          program:
+            title: Program Information
+            tag: asdf://stsci.edu/datamodels/roman/tags/program-1.0.0
+        required: [basic, program]
+  data:
+    title: Segmentation map
+    description: |
+       Segmentation map of an image model, zeros correspond to background.
+    tag: tag:stsci.edu:asdf/core/ndarray-1.*
+    ndim: 2
+    datatype: uint32
+    exact_datatype: true
+
+required: [meta, data]
+flowStyle: block
+propertyOrder: [meta, data]
+...