DAS-2373: Adds support to drop unprocessable variables. (#38)

Author: flamingbear

CHANGELOG.md

@@ -4,6 +4,14 @@ The Harmony Regridding Service follows semantic versioning. All notable changes
 to this project will be documented in this file. The format is based on [Keep a
 Changelog](http://keepachangelog.com/en/1.0.0/).
 
+## [v1.6.0] - 2025-09-12
+
+### Changed
+
+- Unprocessable variables are excluded from output. These include any string
+  variable or any variable configured for exclusion via earthata-varinfo's
+  ExcludedScienceVariables.
+
 ## [v1.5.0] - 2025-08-11
 
 ### Changed
@@ -136,6 +144,8 @@ include updated documentation and files outlined by the
 For more information on internal releases prior to NASA open-source approval,
 see legacy-CHANGELOG.md.
 
+[v1.6.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.6.0
+[v1.5.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.5.0
 [v1.4.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.4.0
 [v1.3.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.3.0
 [v1.2.0]: https://github.com/nasa/harmony-regridding-service/releases/tag/1.2.0

docker/service_version.txt

@@ -1 +1 @@
-1.5.0
+1.6.0

harmony_regridding_service/exceptions.py

@@ -64,3 +64,11 @@ class InvalidCRSResampling(RegridderException):
 
     def __init__(self, message: str):
         super().__init__(message)
+
+
+class InvalidVariableRequest(RegridderException):
+    """Raised when a user requests an unprocessable variable."""
+
+    def __init__(self, bad_vars: set[str]):
+        message = f'Request for unprocessable variable(s): {bad_vars}.'
+        super().__init__(message)

harmony_regridding_service/regridding_service.py

@@ -18,6 +18,7 @@
 )
 from harmony_regridding_service.exceptions import (
     InvalidCRSResampling,
+    InvalidVariableRequest,
 )
 from harmony_regridding_service.file_io import (
     clone_variables,
@@ -33,12 +34,14 @@
     transfer_resampled_dimensions,
     unresampled_variables,
 )
+from harmony_regridding_service.var_utilitities import get_unprocessable_variables
 
 logger = getLogger(__name__)
 
-HRS_VARINFO_CONFIG_FILENAME = str(
-    Path(Path(__file__).parent, 'config', 'HRS_varinfo_config.json')
-)
+
+def varinfo_config_filename() -> str:
+    """Return a path to the varinfo config."""
+    return str(Path(Path(__file__).parent, 'config', 'HRS_varinfo_config.json'))
 
 
 def regrid(
@@ -53,10 +56,12 @@ def regrid(
     logger.info(f'Format:\n {message.format}')
     logger.info(f'Source:\n {source}')
 
+    user_requested_variables = {f'/{v.name.lstrip("/")}' for v in source.variables}
+
     var_info = VarInfoFromNetCDF4(
         input_filepath,
         short_name=source.shortName,  # pyright: ignore[reportAttributeAccessIssue]
-        config_file=HRS_VARINFO_CONFIG_FILENAME,
+        config_file=varinfo_config_filename(),
     )
 
     try:
@@ -81,12 +86,14 @@ def regrid(
 
         vars_to_process = var_info.get_all_variables()
         unresampled_vars = unresampled_variables(var_info)
+
         grid_mapping_variable_names = filter_grid_mappings_to_variables(
             input_grid_mappings(source_ds, vars_to_process)
         )
 
-        logger.info(f'dropping grid_mappings: {grid_mapping_variable_names}')
-        vars_to_process -= grid_mapping_variable_names
+        if grid_mapping_variable_names:
+            logger.info(f'dropping grid_mappings: {grid_mapping_variable_names}')
+            vars_to_process -= grid_mapping_variable_names
 
         cloned_vars = clone_variables(
             source_ds, target_ds, unresampled_vars - grid_mapping_variable_names
@@ -100,8 +107,17 @@ def regrid(
         logger.info(f'processed dimension variables: {dimension_vars}')
         vars_to_process -= dimension_vars
 
+        unprocessable_variables = get_unprocessable_variables(var_info, vars_to_process)
+        if unprocessable_variables:
+            if unprocessable_variables.intersection(user_requested_variables):
+                raise InvalidVariableRequest(
+                    unprocessable_variables.intersection(user_requested_variables)
+                )
+            logger.info(f'Dropping unprocessable variables: {unprocessable_variables}')
+            vars_to_process -= unprocessable_variables
+
         resampled_vars = resample_n_dimensional_variables(
-            source_ds, target_ds, var_info, resampler_cache, set(vars_to_process)
+            source_ds, target_ds, var_info, resampler_cache, vars_to_process
         )
         vars_to_process -= resampled_vars
         logger.info(f'resampled variables: {resampled_vars}')

harmony_regridding_service/var_utilitities.py

@@ -0,0 +1,41 @@
+"""Variable helper functions.
+
+A collection of functions for getting information about variables, mostly with
+varinfo.
+
+"""
+
+import re
+
+from varinfo import VarInfoFromNetCDF4
+
+
+def get_unprocessable_variables(
+    var_info: VarInfoFromNetCDF4, var_list: set[str]
+) -> set[str]:
+    """Variables that can't be processed by Harmony Regridding Service.
+
+    Currently string variables are unprocessable because pyresample cannot
+    handle them. All string variables are excluded.
+
+    Science variables that are excluded explicitly by varInfo are also removed
+    """
+    string_vars = {var for var in var_list if is_string_variable(var_info, var)}
+    excluded_vars = {
+        var for var in var_list if is_excluded_science_variable(var_info, var)
+    }
+
+    return string_vars | excluded_vars
+
+
+def is_excluded_science_variable(var_info: VarInfoFromNetCDF4, var) -> bool:
+    """Returns True if variable is explicitly excluded by VarInfo configuration."""
+    exclusions_pattern = re.compile(
+        '|'.join(var_info.cf_config.excluded_science_variables)
+    )
+    return var_info.variable_is_excluded(var, exclusions_pattern)
+
+
+def is_string_variable(var_info: VarInfoFromNetCDF4, var_name: str) -> bool:
+    """Returns True if variable is a string type."""
+    return var_info.get_variable(var_name).data_type in ['str', 'bytes8']

pip_requirements.txt

@@ -1,9 +1,9 @@
 # Open source packages available from PyPI
 dask~=2025.2.0
-earthdata-varinfo~=3.1.0
-harmony-service-lib~=2.6.0
+earthdata-varinfo~=3.3.0
+harmony-service-lib~=2.7.1
 netCDF4~=1.7.2
 numpy~=2.2.4
 pyproj~=3.7.1
 pyresample~=1.34.0
-xarray~=2025.3.1
+xarray~=2025.9.0

tests/unit/fixtures/test_HRS_varinfo_config.json

@@ -0,0 +1,19 @@
+{
+  "Identification": "harmony_regridding_service_test_varinfo_config",
+  "Version": 1,
+  "CollectionShortNamePath": [ ],
+  "Mission": {
+    "SPL[34].+": "SMAP"
+  },
+  "ExcludedScienceVariables": [
+    {
+      "Applicability": {
+        "Mission": "SMAP"
+      },
+      "VariablePattern": [
+	"/.*altitude_dem.*"
+      ]
+    }
+  ],
+  "MetadataOverrides": []
+}

tests/unit/test_regridding_service.py

@@ -1,5 +1,6 @@
 """Tests the regridding service module."""
 
+import logging
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
@@ -8,6 +9,7 @@
 from harmony_service_lib.message import Message as HarmonyMessage
 from harmony_service_lib.message import Source as HarmonySource
 
+from harmony_regridding_service.exceptions import InvalidVariableRequest
 from harmony_regridding_service.regridding_service import regrid
 
 test_scale_extent = {
@@ -36,7 +38,7 @@ def test_regrid_projected_data_end_to_end(
     """Test the full regrid process for projected input data."""
     input_filename = str(smap_projected_netcdf_file)
     output_filename = str(tmp_path / 'regridded_output.nc')
-    logger_mock = MagicMock()
+    logger = logging.getLogger()
 
     # Define a target CRS [and optionally grid parameters]
     params = {
@@ -57,7 +59,7 @@ def test_regrid_projected_data_end_to_end(
         'harmony_regridding_service.regridding_service.generate_output_filename',
         return_value=output_filename,
     ):
-        result_filename = regrid(message, input_filename, source, logger_mock)
+        result_filename = regrid(message, input_filename, source, logger)
 
     assert result_filename == output_filename
     assert Path(output_filename).exists()
@@ -82,14 +84,14 @@ def test_regrid_projected_data_end_to_end(
         ), description
 
 
-def test_regrid_smap_data_end_to_end(
+def test_regrid_smap_file(
     test_spl3ftp_ncfile,
     tmp_path,
 ):
     """Test the full regrid process for projected input data."""
     input_filename = str(test_spl3ftp_ncfile)
     output_filename = str(tmp_path / 'regridded_output.nc')
-    logger_mock = MagicMock()
+    logger = logging.getLogger()
 
     # Define a target CRS [and optionally grid parameters]
     params = {
@@ -107,7 +109,7 @@ def test_regrid_smap_data_end_to_end(
         'harmony_regridding_service.regridding_service.generate_output_filename',
         return_value=output_filename,
     ):
-        result_filename = regrid(message, input_filename, source, logger_mock)
+        result_filename = regrid(message, input_filename, source, logger)
 
     assert result_filename == output_filename
     assert Path(output_filename).exists()
@@ -139,3 +141,146 @@ def test_regrid_smap_data_end_to_end(
             assert 'longitude' in dt[group], f'failed: {group}'
             assert 'latitude' in dt[group], f'failed: {group}'
             assert 'altitude_dem' in dt[group], f'failed: {group}'
+
+
+def test_regrid_smap_excluded_variable_file(
+    test_spl3ftp_ncfile,
+    tmp_path,
+):
+    """Test the full regrid process with excluded variables.
+
+    This test adds a test varinfo config that excludes science variables:
+    "/.*altitude_dem.*"
+
+    Is it the same as the previous test, but the last assertion is that the
+    variable is not in the output.
+
+    """
+    input_filename = str(test_spl3ftp_ncfile)
+    output_filename = str(tmp_path / 'regridded_output.nc')
+    logger = logging.getLogger()
+
+    # Define a target CRS [and optionally grid parameters]
+    params = {
+        'format': {
+            'mime': 'application/x-netcdf',
+            'crs': 'EPSG:4326',
+        },
+        'sources': [{'collection': 'C123-test', 'shortName': 'SPL3FTP'}],
+    }
+    message = HarmonyMessage(params)
+    source = HarmonySource({'collection': 'C123-TEST', 'shortName': 'SPL3FTP'})
+
+    # Mock generate_output_filename to control the output path
+    with (
+        patch(
+            'harmony_regridding_service.regridding_service.generate_output_filename',
+            return_value=output_filename,
+        ),
+        patch(
+            'harmony_regridding_service.regridding_service.varinfo_config_filename',
+            return_value=str(
+                Path(Path(__file__).parent / 'fixtures/test_HRS_varinfo_config.json')
+            ),
+        ),
+    ):
+        result_filename = regrid(message, input_filename, source, logger)
+
+    assert result_filename == output_filename
+    assert Path(output_filename).exists()
+
+    expected_groups = [
+        '/Freeze_Thaw_Retrieval_Data_Polar',
+        '/Freeze_Thaw_Retrieval_Data_Global',
+    ]
+    expected = {
+        '/Freeze_Thaw_Retrieval_Data_Polar': {
+            'width': 263,
+            'height': 122,
+        },
+        '/Freeze_Thaw_Retrieval_Data_Global': {
+            'width': 186,
+            'height': 73,
+        },
+    }
+
+    with xr.open_datatree(output_filename) as dt:
+        for group in expected_groups:
+            expects = expected[group]
+
+            assert 'crs' in dt[group], f'failed: {group}'
+
+            assert dt[group].dims['y'] == expects['height'], f'failed: {group}'
+            assert dt[group].dims['x'] == expects['width'], f'failed: {group}'
+
+            assert 'longitude' in dt[group], f'failed: {group}'
+            assert 'latitude' in dt[group], f'failed: {group}'
+            ### This is the change from the previous test. altitude_dem is
+            ### configured to be an excluded science variable
+
+            assert 'altitude_dem' not in dt[group], f'failed: {group}'
+
+
+def test_regrid_smap_bad_user_requested_variable_data_end_to_end(
+    test_spl3ftp_ncfile,
+    tmp_path,
+):
+    """Test a Request that specifies an explicitly excluded variable.
+
+    This test repeats the previous test but a new test config is used that
+    explicitly excludes that user's variable. we expect this request to fail.
+
+    """
+    input_filename = str(test_spl3ftp_ncfile)
+    output_filename = str(tmp_path / 'regridded_output.nc')
+    logger = MagicMock()
+
+    # Define a user selected variable
+    user_var = {
+        'id': 'V12345789-EEDTEST',
+        'name': 'Freeze_Thaw_Retrieval_Data_Global/altitude_dem',
+    }
+
+    # Define a target CRS [and optionally grid parameters]
+    params = {
+        'format': {
+            'mime': 'application/x-netcdf',
+            'crs': 'EPSG:4326',
+        },
+        'sources': [
+            {'collection': 'C123-test', 'shortName': 'SPL3FTP', 'variables': [user_var]}
+        ],
+    }
+    message = HarmonyMessage(params)
+    source = HarmonySource(message['sources'][0])
+
+    # Mock generate_output_filename to control the output path
+    with (
+        patch(
+            'harmony_regridding_service.regridding_service.generate_output_filename',
+            return_value=output_filename,
+        ),
+        patch(
+            'harmony_regridding_service.regridding_service.varinfo_config_filename',
+            return_value=str(
+                Path(Path(__file__).parent / 'fixtures/test_HRS_varinfo_config.json')
+            ),
+        ),
+    ):
+        # when harmony-service-lib is updated use this.
+        # expected_message = re.escape(
+        #     r"Request for unprocessable variable(s): "
+        #     "{'/Freeze_Thaw_Retrieval_Data_Global/altitude_dem'}."
+        # )
+        # with pytest.raises(InvalidVariableRequest, match=expected_message):
+        #     regrid(message, input_filename, source, logger)
+
+        expected_message = (
+            'Request for unprocessable variable(s): '
+            "{'/Freeze_Thaw_Retrieval_Data_Global/altitude_dem'}."
+        )
+
+        with pytest.raises(InvalidVariableRequest) as exception_info:
+            regrid(message, input_filename, source, logger)
+
+        assert exception_info.value.message == expected_message