allow embedded access of values

Author: jdkent

store/backend/neurostore/resources/data.py

@@ -599,6 +599,30 @@ def _extract_analysis_id(note):
             return analysis.get("id")
         return analysis
 
+    @staticmethod
+    def _collect_column_values(data, column):
+        collected = []
+
+        def _walk(obj):
+            if isinstance(obj, dict):
+                if column in obj and not isinstance(obj[column], (dict, list)):
+                    collected.append(obj[column])
+                for value in obj.values():
+                    _walk(value)
+            elif isinstance(obj, list):
+                dict_items = [item for item in obj if isinstance(item, dict)]
+                if dict_items and all(column in item for item in dict_items):
+                    for item in dict_items:
+                        value = item.get(column)
+                        if not isinstance(value, (dict, list)):
+                            collected.append(value)
+                else:
+                    for item in obj:
+                        _walk(item)
+
+        _walk(data)
+        return collected
+
     def _normalize_pipeline_specs(self, pipelines):
         specs = []
         column_counter = Counter()
@@ -803,6 +827,7 @@ def _fetch_pipeline_data(self, spec, base_study_ids):
                 )
                 per_base[row.base_study_id] = {
                     "flat": flattened,
+                    "raw": result_data if isinstance(result_data, dict) else {},
                     "config_id": row.config_id,
                     "version": row.version,
                     "timestamp": timestamp,
@@ -881,6 +906,24 @@ def _apply_pipeline_columns(self, annotation, data, specs, column_counter):
                     entry = pipeline_data.get(base_study_id)
                     flat_values = entry["flat"] if entry else {}
                     value = flat_values.get(column)
+                    if isinstance(value, list):
+                        flattened_value = ",".join(
+                            str(item) for item in value if item is not None
+                        )
+                        value = flattened_value if flattened_value else None
+                    if value is None and entry:
+                        raw_values = [
+                            v
+                            for v in self._collect_column_values(
+                                entry.get("raw", {}), column
+                            )
+                            if v is not None
+                        ]
+                        if raw_values:
+                            if len(raw_values) == 1:
+                                value = str(raw_values[0])
+                            else:
+                                value = ",".join(str(v) for v in raw_values)
 
                     payload.setdefault("note", {})
                     payload["note"][key_name] = value

store/backend/neurostore/tests/api/test_annotations.py

@@ -488,7 +488,14 @@ def test_put_annotation_applies_pipeline_columns(auth_client, session):
         config=config,
         status="SUCCESS",
         date_executed=datetime(2024, 1, 1, tzinfo=timezone.utc),
-        result_data={"string_field": "demo", "numeric_field": 42},
+        result_data={
+            "string_field": "demo",
+            "numeric_field": 42,
+            "array_field": [
+                {"name": "a"},
+                {"name": "b"},
+            ],
+        },
         file_inputs={},
     )
     session.add_all([pipeline, config, result])
@@ -498,7 +505,7 @@ def test_put_annotation_applies_pipeline_columns(auth_client, session):
         "pipelines": [
             {
                 "name": pipeline.name,
-                "columns": ["string_field", "numeric_field"],
+                "columns": ["string_field", "numeric_field", "name"],
             }
         ]
     }
@@ -510,6 +517,7 @@ def test_put_annotation_applies_pipeline_columns(auth_client, session):
     assert body["note_keys"]["existing"] == "string"
     assert body["note_keys"]["string_field"] == "string"
     assert body["note_keys"]["numeric_field"] == "number"
+    assert body["note_keys"]["name"] == "string"
 
     notes = body["notes"]
     assert len(notes) == 2
@@ -518,6 +526,7 @@ def test_put_annotation_applies_pipeline_columns(auth_client, session):
         assert note["existing"] in {"A1", "A2"}
         assert note["string_field"] == "demo"
         assert note["numeric_field"] == 42
+        assert note["name"] == "a,b"
 
 
 def test_put_annotation_pipeline_column_conflict_suffix(auth_client, session):
@@ -536,7 +545,13 @@ def test_put_annotation_pipeline_column_conflict_suffix(auth_client, session):
         config=config_one,
         status="SUCCESS",
         date_executed=datetime(2024, 1, 1, tzinfo=timezone.utc),
-        result_data={"string_field": "primary"},
+        result_data={
+            "string_field": "primary",
+            "array_field": [
+                {"name": "a"},
+                {"name": "b"},
+            ],
+        },
         file_inputs={},
     )
 
@@ -570,7 +585,7 @@ def test_put_annotation_pipeline_column_conflict_suffix(auth_client, session):
 
     payload = {
         "pipelines": [
-            {"name": pipeline_one.name, "columns": ["string_field"]},
+            {"name": pipeline_one.name, "columns": ["string_field", "name"]},
             {
                 "name": pipeline_two.name,
                 "columns": ["string_field"],
@@ -590,11 +605,13 @@ def test_put_annotation_pipeline_column_conflict_suffix(auth_client, session):
     assert key_two in body["note_keys"]
     assert body["note_keys"][key_one] == "string"
     assert body["note_keys"][key_two] == "string"
+    assert body["note_keys"]["name"] == "string"
 
     for entry in body["notes"]:
         note = entry["note"]
         assert note[key_one] == "primary"
         assert note[key_two] == "secondary"
+        assert note["name"] == "a,b"
 
 
 def test_annotation_analyses_post(auth_client, ingest_neurosynth, session):
@@ -629,8 +646,11 @@ def test_annotation_analyses_post(auth_client, ingest_neurosynth, session):
     get_resp = auth_client.get(f"/api/annotations/{annot.json()['id']}")
 
     assert len(post_resp.json()) == 2  # third input did not have proper id
-    assert (
-        get_resp.json()["notes"][1]["note"]["doo"]
-        == post_resp.json()[1]["note"]["doo"]
-        == new_value
-    )
+    updated_by_analysis = {
+        note["analysis"]: note["note"]["doo"] for note in post_resp.json()
+    }
+    current_by_analysis = {
+        note["analysis"]: note["note"]["doo"] for note in get_resp.json()["notes"]
+    }
+    for analysis_id, value in updated_by_analysis.items():
+        assert current_by_analysis[analysis_id] == value == new_value