wip: add order

Author: jdkent

store/backend/neurostore/ingest/__init__.py

@@ -28,6 +28,7 @@
     Entity,
 )
 from neurostore.models.data import StudysetStudy, _check_type
+from neurostore.utils import normalize_note_keys
 
 META_ANALYSIS_WORDS = ["meta analysis", "meta-analysis", "systematic review"]
 
@@ -356,9 +357,9 @@ def ingest_neurosynth(max_rows=None):
                 notes.append(aa)
 
         # add notes to annotation
-        annot.note_keys = {
-            k: _check_type(v) for k, v in annotation_row._asdict().items()
-        }
+        annot.note_keys = normalize_note_keys(
+            {k: _check_type(v) for k, v in annotation_row._asdict().items()}
+        )
         annot.annotation_analyses = notes
         for note in notes:
             to_commit.append(note.analysis)

store/backend/neurostore/models/data.py

@@ -14,6 +14,7 @@
 from .migration_types import TSVector, VectorType
 from ..database import db
 from ..utils import parse_json_filter, build_jsonpath
+from ..utils import normalize_note_keys
 
 # status of pipeline run
 STATUS_ENUM = PGEnum(
@@ -129,6 +130,12 @@ class Annotation(BaseMixin, db.Model):
         cascade_backrefs=False,
     )
 
+    @validates("note_keys")
+    def _validate_note_keys(self, key, value):
+        if isinstance(value, dict):
+            return normalize_note_keys(value)
+        return value
+
 
 class AnnotationAnalysis(db.Model):
     __tablename__ = "annotation_analyses"

store/backend/neurostore/resources/data.py

@@ -51,7 +51,7 @@
 )
 from ..models.data import StudysetStudy, BaseStudy, _check_type
 from ..utils import parse_json_filter, build_jsonpath
-
+from ..utils import normalize_note_keys
 
 from ..schemas import (
     BooleanOrString,
@@ -330,7 +330,9 @@ def _clone_annotations(self, source_record, cloned_record):
                 ),
                 public=annotation.public,
                 note_keys=(
-                    deepcopy(annotation.note_keys) if annotation.note_keys else {}
+                    normalize_note_keys(annotation.note_keys)
+                    if annotation.note_keys
+                    else {}
                 ),
             )
             clone_annotation.studyset = cloned_record
@@ -944,8 +946,13 @@ def _apply_pipeline_columns(self, annotation, data, specs, column_counter):
             else:
                 note_keys = dict(data["note_keys"])
             for key, value_type in column_types.items():
-                note_keys[key] = value_type or "string"
-            data["note_keys"] = note_keys
+                existing_value = note_keys.get(key)
+                normalized_type = value_type or "string"
+                if isinstance(existing_value, dict):
+                    existing_value["type"] = normalized_type
+                else:
+                    note_keys[key] = {"type": normalized_type}
+            data["note_keys"] = normalize_note_keys(note_keys)
 
         data["annotation_analyses"] = list(note_map.values())
 

store/backend/neurostore/schemas/data.py

@@ -16,6 +16,7 @@
 
 from neurostore.core import db
 from neurostore.models import Analysis, Point
+from neurostore.utils import normalize_note_keys
 
 # context parameters
 # clone: create a new object with new ids (true or false)
@@ -667,6 +668,12 @@ class AnnotationSchema(BaseDataSchema):
         fields.Nested(AnnotationPipelineSchema), load_only=True, required=False
     )
 
+    @pre_load
+    def normalize_note_keys_field(self, data, **kwargs):
+        if data.get("note_keys") is not None:
+            data["note_keys"] = normalize_note_keys(data["note_keys"])
+        return data
+
     @pre_load
     def add_studyset_id(self, data, **kwargs):
         if data.get("studyset") and data.get("notes"):
@@ -694,6 +701,12 @@ def validate_notes(self, data, **kwargs):
         if invalid:
             raise ValidationError({"notes": invalid})
 
+    @post_dump
+    def ensure_note_keys_order(self, data, **kwargs):
+        if data.get("note_keys"):
+            data["note_keys"] = normalize_note_keys(data["note_keys"])
+        return data
+
 
 class BaseSnapshot(object):
     def __init__(self):

store/backend/neurostore/tests/api/test_annotations.py

@@ -15,6 +15,16 @@
 )
 
 
+def _assert_note_keys(note_keys, expected_entries):
+    keys = list(note_keys.keys())
+    expected_keys = [key for key, _ in expected_entries]
+    assert keys == expected_keys
+    for order, (key, expected_type) in enumerate(expected_entries):
+        entry = note_keys[key]
+        assert entry["type"] == expected_type
+        assert entry["order"] == order
+
+
 def _create_annotation_with_two_analyses(session, user):
     base_study = BaseStudy(name="Test Base Study", level="group", user=user)
     study = Study(
@@ -514,10 +524,15 @@ def test_put_annotation_applies_pipeline_columns(auth_client, session):
     assert resp.status_code == 200
     body = resp.json()
 
-    assert body["note_keys"]["existing"] == "string"
-    assert body["note_keys"]["string_field"] == "string"
-    assert body["note_keys"]["numeric_field"] == "number"
-    assert body["note_keys"]["name"] == "string"
+    _assert_note_keys(
+        body["note_keys"],
+        [
+            ("existing", "string"),
+            ("string_field", "string"),
+            ("numeric_field", "number"),
+            ("name", "string"),
+        ],
+    )
 
     notes = body["notes"]
     assert len(notes) == 2
@@ -601,11 +616,15 @@ def test_put_annotation_pipeline_column_conflict_suffix(auth_client, session):
     key_one = f"string_field_{pipeline_one.name}_{config_one.version}_{config_one.id}"
     key_two = f"string_field_{pipeline_two.name}_{config_two.version}_{config_two.id}"
 
-    assert key_one in body["note_keys"]
-    assert key_two in body["note_keys"]
-    assert body["note_keys"][key_one] == "string"
-    assert body["note_keys"][key_two] == "string"
-    assert body["note_keys"]["name"] == "string"
+    _assert_note_keys(
+        body["note_keys"],
+        [
+            ("existing", "string"),
+            (key_one, "string"),
+            ("name", "string"),
+            (key_two, "string"),
+        ],
+    )
 
     for entry in body["notes"]:
         note = entry["note"]

store/backend/neurostore/utils.py

@@ -3,6 +3,8 @@
 """
 
 import re
+from copy import deepcopy
+from typing import Any, Dict, Iterable, Tuple
 
 
 def determine_value_type(value: str):
@@ -203,3 +205,84 @@ def parse_json_filter(filter_str: str) -> tuple:
             raise ValueError(f"Invalid numeric value '{value}' for operator {operator}")
 
     return pipeline_name, version, field_path, operator, value
+
+
+DEFAULT_NOTE_KEY_TYPE = "string"
+
+
+def _extract_note_key_type(value: Any) -> str:
+    """Derive the type string from a note_keys entry."""
+    if isinstance(value, dict):
+        for candidate in ("type", "value_type", "data_type", "datatype"):
+            candidate_value = value.get(candidate)
+            if candidate_value is not None:
+                return str(candidate_value)
+        return DEFAULT_NOTE_KEY_TYPE
+    if value is None:
+        return DEFAULT_NOTE_KEY_TYPE
+    return str(value)
+
+
+def _hydrate_note_key_entry(
+    key: str, value: Any, original_index: int
+) -> Tuple[str, Dict[str, Any], int, int]:
+    """
+    Build a normalized representation of a note_key entry while delaying order assignment.
+
+    Returns a tuple of:
+      - key name
+      - entry dict (deep copied when applicable)
+      - provided order (or None)
+      - original index (for stability when order is missing)
+    """
+    if isinstance(value, dict):
+        entry = deepcopy(value)
+        provided_order = entry.get("order")
+    else:
+        entry = {}
+        provided_order = None
+
+    entry["type"] = _extract_note_key_type(value)
+    if provided_order is not None:
+        try:
+            provided_order = int(provided_order)
+        except (ValueError, TypeError):
+            provided_order = None
+
+    return key, entry, provided_order, original_index
+
+
+def normalize_note_keys(note_keys: Any) -> Dict[str, Dict[str, Any]]:
+    """
+    Ensure note_keys are stored as an ordered mapping of key -> metadata, where
+    metadata includes a ``type`` and zero-based ``order`` attribute.
+
+    The returned mapping preserves any additional attributes present on each
+    entry and reindexes orders sequentially starting at 0. When no explicit
+    order is supplied, the original insertion order is used.
+    """
+    if not isinstance(note_keys, dict) or not note_keys:
+        return {}
+
+    hydrated_entries: Iterable[Tuple[str, Dict[str, Any], int, int]] = [
+        _hydrate_note_key_entry(key, value, original_index)
+        for original_index, (key, value) in enumerate(note_keys.items())
+    ]
+
+    sorted_entries = sorted(
+        hydrated_entries,
+        key=lambda item: (
+            item[2] is None,
+            item[2] if item[2] is not None else item[3],
+            item[0],
+        ),
+    )
+
+    normalized: Dict[str, Dict[str, Any]] = {}
+    for new_order, (key, entry, _provided_order, _original_index) in enumerate(
+        sorted_entries
+    ):
+        entry["order"] = new_order
+        normalized[key] = entry
+
+    return normalized