Add regression tests for nullable_nonnegative decode filter in write_tables (#1087)

* adding nullable_nonnegative decode filter

* Initial plan

* Add unit tests for _decode_output_column and _apply_decode_filter (nullable_nonnegative)

---------

Co-authored-by: David Hensle <51132108+dhensle@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: Copilot

activitysim/core/configuration/top.py

@@ -114,6 +114,9 @@ class OutputTable(PydanticBase):
     should be decoded, as negative values indicate an absence of choice.  In
     these cases, the "tablename.fieldname" can be prefixed with a "nonnegative"
     filter, seperated by a pipe character (e.g. "nonnegative | land_use.zone_id").
+    If the column may also contain null values, use "nullable_nonnegative"
+    instead to pass nulls through unchanged while still preserving negative
+    sentinel values.
     """
 
 

activitysim/core/steps/_decode.py

@@ -0,0 +1,40 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def _decode_output_column(column, map_func, preserve_nulls=False):
+    series = pd.Series(column)
+    if preserve_nulls:
+        revised_col = series.copy()
+        notna = series.notna()
+        revised_col.loc[notna] = series.loc[notna].astype(int).map(map_func)
+        return revised_col
+    return series.astype(int).map(map_func)
+
+
+def _apply_decode_filter(map_col, decode_filter):
+    preserve_nulls = False
+
+    if decode_filter is None:
+        return map_col.__getitem__, preserve_nulls
+
+    if decode_filter == "nonnegative":
+
+        def map_func(x):
+            return x if x < 0 else map_col[x]
+
+        return map_func, preserve_nulls
+
+    if decode_filter == "nullable_nonnegative":
+        preserve_nulls = True
+
+        def map_func(x):
+            return x if x < 0 else map_col[x]
+
+        return map_func, preserve_nulls
+
+    raise ValueError(f"unknown decode_filter {decode_filter}")

activitysim/core/steps/output.py

@@ -17,6 +17,7 @@
 from activitysim.core import configuration, workflow
 from activitysim.core.workflow.checkpoint import CHECKPOINT_NAME
 from activitysim.core.estimation import estimation_enabled, EstimationConfig
+from activitysim.core.steps._decode import _apply_decode_filter, _decode_output_column
 
 logger = logging.getLogger(__name__)
 
@@ -515,9 +516,11 @@ def write_tables(state: workflow.State) -> None:
 
                 if decode_instruction == "time_period":
                     map_col = list(state.network_settings.skim_time_periods.labels)
-                    map_func = map_col.__getitem__
-                    revised_col = (
-                        pd.Series(dt.column(colname)).astype(int).map(map_func)
+                    map_func, preserve_nulls = _apply_decode_filter(
+                        map_col, decode_filter
+                    )
+                    revised_col = _decode_output_column(
+                        dt.column(colname), map_func, preserve_nulls=preserve_nulls
                     )
                     dt = dt.drop([colname]).append_column(
                         colname, pa.array(revised_col)
@@ -536,18 +539,10 @@ def write_tables(state: workflow.State) -> None:
                 except KeyError:
                     map_col = parent_table.column(lookup_col)
                 map_col = np.asarray(map_col)
-                map_func = map_col.__getitem__
-                if decode_filter:
-                    if decode_filter == "nonnegative":
-
-                        def map_func(x):
-                            return x if x < 0 else map_col[x]
-
-                    else:
-                        raise ValueError(f"unknown decode_filter {decode_filter}")
+                map_func, preserve_nulls = _apply_decode_filter(map_col, decode_filter)
                 if colname in dt.column_names:
-                    revised_col = (
-                        pd.Series(dt.column(colname)).astype(int).map(map_func)
+                    revised_col = _decode_output_column(
+                        dt.column(colname), map_func, preserve_nulls=preserve_nulls
                     )
                     dt = dt.drop([colname]).append_column(
                         colname, pa.array(revised_col)

activitysim/core/test/test_output.py

@@ -0,0 +1,83 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import pandas.testing as pdt
+import pytest
+
+from activitysim.core.steps._decode import _apply_decode_filter, _decode_output_column
+
+
+@pytest.fixture
+def zone_labels():
+    """Sample zone ID lookup array (index 0..4 -> zone labels)."""
+    return np.array([100, 200, 300, 400, 500])
+
+
+def test_decode_output_column_no_nulls(zone_labels):
+    """Basic decode without nulls: each integer index maps to the zone label."""
+    map_func = zone_labels.__getitem__
+    column = pd.array([0, 1, 2, 3, 4])
+    result = _decode_output_column(column, map_func)
+    expected = pd.Series([100, 200, 300, 400, 500])
+    pdt.assert_series_equal(result, expected)
+
+
+def test_decode_output_column_preserve_nulls(zone_labels):
+    """With preserve_nulls=True, NaN entries pass through unchanged."""
+    map_func = zone_labels.__getitem__
+    column = pd.array([0, pd.NA, 2, pd.NA, 4], dtype="Int64")
+    result = _decode_output_column(column, map_func, preserve_nulls=True)
+    # Non-null positions are decoded; null positions remain null.
+    assert result[0] == 100
+    assert result[2] == 300
+    assert result[4] == 500
+    assert pd.isna(result[1])
+    assert pd.isna(result[3])
+
+
+def test_apply_decode_filter_none(zone_labels):
+    """No filter: map_func is direct indexing and preserve_nulls is False."""
+    map_func, preserve_nulls = _apply_decode_filter(zone_labels, None)
+    assert not preserve_nulls
+    assert map_func(2) == zone_labels[2]
+
+
+def test_apply_decode_filter_nonnegative(zone_labels):
+    """nonnegative filter: negative values pass through; others are decoded."""
+    map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nonnegative")
+    assert not preserve_nulls
+    assert map_func(1) == 200
+    assert map_func(-1) == -1
+
+
+def test_apply_decode_filter_nullable_nonnegative(zone_labels):
+    """nullable_nonnegative filter: negative values pass through; preserve_nulls is True."""
+    map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nullable_nonnegative")
+    assert preserve_nulls
+    assert map_func(1) == 200
+    assert map_func(-1) == -1
+
+
+def test_apply_decode_filter_unknown():
+    """Unknown filter name raises ValueError."""
+    with pytest.raises(ValueError, match="unknown decode_filter"):
+        _apply_decode_filter([], "bad_filter")
+
+
+def test_nullable_nonnegative_with_nulls_and_sentinels(zone_labels):
+    """
+    End-to-end: a column with nulls and -1 sentinel values decoded via
+    nullable_nonnegative should pass nulls and negative values through unchanged
+    while mapping non-negative indexes to zone labels.
+    """
+    map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nullable_nonnegative")
+    column = pd.array([0, pd.NA, -1, 3, pd.NA], dtype="Int64")
+    result = _decode_output_column(column, map_func, preserve_nulls=preserve_nulls)
+    assert result[0] == 100
+    assert pd.isna(result[1])
+    assert result[2] == -1
+    assert result[3] == 400
+    assert pd.isna(result[4])

docs/dev-guide/using-sharrow.md

@@ -111,7 +111,9 @@ decoded in various output files.  Generally, for columns that are fully populate
 with zone id's (e.g. tour and trip ends) we can apply the `decode_columns` settings
 to reverse the mapping and restore the nominal zone id's globally for the entire
 column of data.  For columns where there is some missing data flagged by negative
-values, the "nonnegative" filter is prepended to the instruction.
+values, the "nonnegative" filter is prepended to the instruction. If the column
+may also contain nulls, use "nullable_nonnegative" to leave those values
+unchanged while decoding the non-negative indexes.
 
 ```yaml
 output_tables:
@@ -127,7 +129,7 @@ output_tables:
       decode_columns:
         home_zone_id: land_use.zone_id
         school_zone_id: nonnegative | land_use.zone_id
-        workplace_zone_id: nonnegative | land_use.zone_id
+        workplace_zone_id: nullable_nonnegative | land_use.zone_id
     - tablename: tours
       decode_columns:
         origin: land_use.zone_id