🐛 Fix fetching REDCap data.

Author: shnizzedy

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.12.3
+
+### Fixed
+
+- Bug fetching REDCap data introduced in v1.12.1.
+
 ## 1.12.2
 
 ### Fixed

VERSION

@@ -1 +1 @@
-1.12.1
+1.12.3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hbnmigration",
-  "version": "1.12.1",
+  "version": "1.12.3",
   "private": true,
   "description": "HBN data migration monitoring infrastructure with Python and Node.js services",
   "workspaces": [

python_jobs/VERSION

@@ -1 +1 @@
-1.12.1
+1.12.3

python_jobs/src/hbnmigration/from_redcap/from_redcap.py

@@ -387,15 +387,11 @@ def fetch_data(
             filter_conditions = " OR ".join(
                 [f"[{field}] != ''" for field in export_fields.split(",")]
             )
-        else:
-            filter_conditions = " AND ".join(
-                [f"[{field}] != ''" for field in export_fields.split(",")]
+            filter_logic = (
+                f"({filter_logic}) AND ({filter_conditions})"
+                if filter_logic
+                else filter_conditions
             )
-        filter_logic = (
-            f"({filter_logic}) AND ({filter_conditions})"
-            if filter_logic
-            else filter_conditions
-        )
     if filter_logic:
         redcap_participant_data["filterLogic"] = filter_logic
     for key, value in exports.items():

python_jobs/src/tests/test_redcap.py

@@ -1965,3 +1965,61 @@ def test_string_export_fields_still_works(self, mock_fetch_api) -> None:
             fetch_data("fake_token", "field1,field2")
         request_data = mock_fetch_api.call_args[0][2]
         assert request_data.get("fields") == "field1,field2"
+
+
+class TestFetchDataRegression:
+    """Ensure backward compatibility and correct filter logic construction."""
+
+    @patch("hbnmigration.from_redcap.from_redcap.fetch_api_data")
+    @patch("hbnmigration.from_redcap.from_redcap.redcap_variables")
+    def test_fetch_data_does_not_inject_mandatory_and_filters(
+        self, mock_vars: MagicMock, mock_fetch_api: MagicMock
+    ) -> None:
+        """
+        Verify that fetch_data does not automatically inject AND conditions.
+
+        v1.12.1 introduced a regression where requesting multiple fields
+        appended 'AND [field] != ""' for every field, causing REDCap to
+        return zero records (and thus NoData) if any single field was empty.
+
+        Parameters
+        ----------
+        mock_vars
+            Mocked REDCap configuration variables.
+        mock_fetch_api
+            Mocked API execution function.
+
+        """
+        # 1. Setup Mock: REDCap headers
+        mock_vars.headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+        # 2. Define expected behavior:
+        # Even if we ask for field1 and field2, if a record exists with only field1
+        # populated, we expect to get that row back.
+        mock_data = pd.DataFrame([{"record_id": "1", "field1": "val1", "field2": ""}])
+        mock_fetch_api.return_value = mock_data
+
+        # 3. Execution: Request multiple fields
+        # In the BROKEN version, this will inject:
+        # filterLogic: ([field1] != '') AND ([field2] != '')
+        # causing the REAL REDCap API to return nothing, though our mock
+        # specifically tests the logic construction.
+
+        try:
+            result = fetch_data(
+                "fake_token", {"fields": "field1,field2"}, all_or_any="all"
+            )
+        except Exception as e:
+            pytest.fail(f"fetch_data raised an exception: {e}")
+
+        # 4. Assertions on the constructed logic
+        # We need to inspect what was actually sent to the API mock
+        sent_payload = mock_fetch_api.call_args[0][2]
+        filter_logic = sent_payload.get("filterLogic", "")
+
+        assert "AND" not in filter_logic, (
+            "Regression Found: fetch_data is forcing 'AND' filters on all fields: "
+            f"{filter_logic}. This causes records with partial data to be excluded "
+            "from the fetch."
+        )
+        assert not result.empty