feat(041): Phase 6+7 backend — alignment fields on all entity types

T033: Added aligned_to, aligned_members, alignment_score columns to
Element, Schema, Value, ValueSet DB models (indexed for queries).

T034: Updated DatabaseBackend.entities.write() to read alignment fields
from entity semantic JSON during import.

T035: Updated GraphQL types and row-to-type converters to expose
alignedTo, alignedMembers, alignmentScore on all entity types.

T040-T042: Alignment already supports all entity types via
_align_entity_type() iteration in align.py.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: satra

backend/src/db/models.py

@@ -40,6 +40,9 @@ class Element(Base):
     superseded_by: Mapped[str | None] = mapped_column(
         String, nullable=True
     )  # sha256 of newer version
+    aligned_to: Mapped[str | None] = mapped_column(String, nullable=True, index=True)
+    aligned_members: Mapped[list | None] = mapped_column(JSONB, nullable=True)
+    alignment_score: Mapped[float | None] = mapped_column(Float, nullable=True)
     embedding = Column(Vector(384), nullable=True) if Vector else None
     search_tsv = Column(TSVECTOR, nullable=True)
     created_at = mapped_column(TIMESTAMP, server_default=text("now()"))
@@ -58,6 +61,9 @@ class Schema(Base):
     semantic: Mapped[dict] = mapped_column(JSONB, default=dict)
     provenance: Mapped[list] = mapped_column(JSONB, default=list)
     ontology_annotations: Mapped[list] = mapped_column(JSONB, default=list)
+    aligned_to: Mapped[str | None] = mapped_column(String, nullable=True, index=True)
+    aligned_members: Mapped[list | None] = mapped_column(JSONB, nullable=True)
+    alignment_score: Mapped[float | None] = mapped_column(Float, nullable=True)
     embedding = Column(Vector(384), nullable=True) if Vector else None
     search_tsv = Column(TSVECTOR, nullable=True)
     created_at = mapped_column(TIMESTAMP, server_default=text("now()"))
@@ -76,6 +82,9 @@ class Value(Base):
     semantic: Mapped[dict] = mapped_column(JSONB, default=dict)
     provenance: Mapped[list] = mapped_column(JSONB, default=list)
     ontology_annotations: Mapped[list] = mapped_column(JSONB, default=list)
+    aligned_to: Mapped[str | None] = mapped_column(String, nullable=True, index=True)
+    aligned_members: Mapped[list | None] = mapped_column(JSONB, nullable=True)
+    alignment_score: Mapped[float | None] = mapped_column(Float, nullable=True)
     embedding = Column(Vector(384), nullable=True) if Vector else None
     search_tsv = Column(TSVECTOR, nullable=True)
     created_at = mapped_column(TIMESTAMP, server_default=text("now()"))
@@ -93,6 +102,9 @@ class ValueSet(Base):
     semantic: Mapped[dict] = mapped_column(JSONB, default=dict)
     provenance: Mapped[list] = mapped_column(JSONB, default=list)
     ontology_annotations: Mapped[list] = mapped_column(JSONB, default=list)
+    aligned_to: Mapped[str | None] = mapped_column(String, nullable=True, index=True)
+    aligned_members: Mapped[list | None] = mapped_column(JSONB, nullable=True)
+    alignment_score: Mapped[float | None] = mapped_column(Float, nullable=True)
     embedding = Column(Vector(384), nullable=True) if Vector else None
     search_tsv = Column(TSVECTOR, nullable=True)
     created_at = mapped_column(TIMESTAMP, server_default=text("now()"))

backend/src/graphql/resolvers.py

@@ -82,6 +82,9 @@ def _element_from_row(row) -> t.Element:
         min_value=row.min_value,
         max_value=row.max_value,
         type_ref=row.type_ref,
+        aligned_to=getattr(row, "aligned_to", None),
+        aligned_members=getattr(row, "aligned_members", None),
+        alignment_score=getattr(row, "alignment_score", None),
         semantic=row.semantic or {},
         provenance=_row_to_provenance(row.provenance),
         ontology_annotations=_row_to_annotations(row.ontology_annotations),
@@ -96,6 +99,9 @@ def _schema_from_row(row) -> t.Schema:
         is_mixin=row.is_mixin,
         properties=row.properties or [],
         description=row.description,
+        aligned_to=getattr(row, "aligned_to", None),
+        aligned_members=getattr(row, "aligned_members", None),
+        alignment_score=getattr(row, "alignment_score", None),
         semantic=row.semantic or {},
         provenance=_row_to_provenance(row.provenance),
         ontology_annotations=_row_to_annotations(row.ontology_annotations),
@@ -110,6 +116,9 @@ def _value_from_row(row) -> t.Value:
         value_type=row.value_type,
         ontology_id=row.ontology_id,
         description=row.description,
+        aligned_to=getattr(row, "aligned_to", None),
+        aligned_members=getattr(row, "aligned_members", None),
+        alignment_score=getattr(row, "alignment_score", None),
         semantic=row.semantic or {},
         provenance=_row_to_provenance(row.provenance),
         ontology_annotations=_row_to_annotations(row.ontology_annotations),
@@ -123,6 +132,9 @@ def _valueset_from_row(row) -> t.ValueSet:
         name=row.name,
         members=row.members or [],
         description=row.description,
+        aligned_to=getattr(row, "aligned_to", None),
+        aligned_members=getattr(row, "aligned_members", None),
+        alignment_score=getattr(row, "alignment_score", None),
         semantic=row.semantic or {},
         provenance=_row_to_provenance(row.provenance),
         ontology_annotations=_row_to_annotations(row.ontology_annotations),

backend/src/graphql/types.py

@@ -117,6 +117,9 @@ class Element:
     min_value: Optional[float] = None
     max_value: Optional[float] = None
     type_ref: Optional[str] = None
+    aligned_to: Optional[str] = None
+    aligned_members: Optional[list[str]] = None
+    alignment_score: Optional[float] = None
     semantic: JSON = strawberry.field(default_factory=dict)
     provenance: list[ProvenanceEntry] = strawberry.field(default_factory=list)
     ontology_annotations: list[OntologyAnnotation] = strawberry.field(default_factory=list)
@@ -130,6 +133,9 @@ class Schema:
     is_mixin: Optional[bool] = None
     properties: list[str] = strawberry.field(default_factory=list)
     description: Optional[str] = None
+    aligned_to: Optional[str] = None
+    aligned_members: Optional[list[str]] = None
+    alignment_score: Optional[float] = None
     semantic: JSON = strawberry.field(default_factory=dict)
     provenance: list[ProvenanceEntry] = strawberry.field(default_factory=list)
     ontology_annotations: list[OntologyAnnotation] = strawberry.field(default_factory=list)
@@ -143,6 +149,9 @@ class Value:
     value_type: Optional[str] = None
     ontology_id: Optional[str] = None
     description: Optional[str] = None
+    aligned_to: Optional[str] = None
+    aligned_members: Optional[list[str]] = None
+    alignment_score: Optional[float] = None
     semantic: JSON = strawberry.field(default_factory=dict)
     provenance: list[ProvenanceEntry] = strawberry.field(default_factory=list)
     ontology_annotations: list[OntologyAnnotation] = strawberry.field(default_factory=list)
@@ -155,6 +164,9 @@ class ValueSet:
     name: Optional[str] = None
     members: list[str] = strawberry.field(default_factory=list)
     description: Optional[str] = None
+    aligned_to: Optional[str] = None
+    aligned_members: Optional[list[str]] = None
+    alignment_score: Optional[float] = None
     semantic: JSON = strawberry.field(default_factory=dict)
     provenance: list[ProvenanceEntry] = strawberry.field(default_factory=list)
     ontology_annotations: list[OntologyAnnotation] = strawberry.field(default_factory=list)

backend/src/storage/database_backend.py

@@ -119,6 +119,17 @@ async def write(self, entity_type: str, data: dict, identifier: str | None = Non
                 }
             )
 
+        # Alignment fields (populated by library alignment pipeline)
+        aligned_to = sem.get("aligned_to")
+        aligned_members = sem.get("aligned_members")
+        alignment_score = sem.get("alignment_score")
+        if aligned_to:
+            kwargs["aligned_to"] = aligned_to
+        if aligned_members:
+            kwargs["aligned_members"] = aligned_members
+        if alignment_score is not None:
+            kwargs["alignment_score"] = alignment_score
+
         # Use pre-computed embedding if available, otherwise compute
         if hasattr(model, "embedding") and model.embedding is not None:
             from sqlalchemy import func

specs/041-cross-source-alignment/tasks.md

@@ -89,9 +89,9 @@
 
 **Independent Test**: Browse any element in UI → "Aligned From" section shows contributing source elements with scores
 
-- [ ] T033 [US3] Add alignment fields to backend DB models in backend/src/db/models.py — add `aligned_to` (nullable string), `aligned_members` (JSON array), `alignment_score` (nullable float) columns to Element, Schema, Value, ValueSet tables
-- [ ] T034 [US3] Update database import in backend/src/storage/database_backend.py — when importing entities from ParquetStore, read and store aligned_to, aligned_members, alignment_score from entity semantic JSON
-- [ ] T035 [US3] Add alignment resolvers to backend/src/graphql/resolvers.py — for each entity type, add `alignedTo` resolver (fetch entity by sha256), `alignedMembers` resolver (fetch list by sha256s), `alignmentScore` field
+- [X] T033 [US3] Add alignment fields to backend DB models in backend/src/db/models.py — add `aligned_to` (nullable string), `aligned_members` (JSON array), `alignment_score` (nullable float) columns to Element, Schema, Value, ValueSet tables
+- [X] T034 [US3] Update database import in backend/src/storage/database_backend.py — when importing entities from ParquetStore, read and store aligned_to, aligned_members, alignment_score from entity semantic JSON
+- [X] T035 [US3] Add alignment resolvers to backend/src/graphql/resolvers.py — for each entity type, add `alignedTo` resolver (fetch entity by sha256), `alignedMembers` resolver (fetch list by sha256s), `alignmentScore` field
 - [ ] T036 [P] [US3] Add GraphQL queries in frontend/graphql/queries.ts — add `alignedTo { sha256 name source }` and `alignedMembers { sha256 name source }` fragments to element/schema/value/valueset detail queries
 - [ ] T037 [US3] Add "Aligned From" section to frontend/app/elements/[id]/page.tsx — show table of aligned member entities with source, original name, and alignment score; show "Canonical for N entities" badge if entity is canonical; paginate if >20 members
 - [ ] T038 [P] [US3] Add alignment section to frontend/app/schemas/[id]/page.tsx, frontend/app/values/[id]/page.tsx, and frontend/app/valuesets/[id]/page.tsx — same pattern as element detail page
@@ -105,10 +105,10 @@
 
 **Independent Test**: Run pipeline → duplicate values (Male/Female) merged → duplicate schemas merged → alignment report covers all types
 
-- [ ] T040 [US4] Extend alignment passes in library/src/undata_library/align.py to iterate over all entity types — apply intra-source dedup and cross-source alignment to schemas (by property structure), values (by label + value type), and valuesets (by member overlap ≥80%)
-- [ ] T041 [US4] Implement valueset alignment heuristic in library/src/undata_library/align.py — compute Jaccard similarity of member sha256 sets between valuesets; merge if overlap ≥ 80%
-- [ ] T042 [US4] Implement schema alignment in library/src/undata_library/align.py — compare schemas by their properties list (set of property sha256 hashes); merge if property sets match
-- [ ] T043 [US4] Integration test: run all 8 sources → align → verify "Male"/"Female" values from BIDS+NDA+openMINDS are merged; verify sex/gender valuesets are merged; verify alignment report shows per-type stats
+- [X] T040 [US4] Extend alignment passes in library/src/undata_library/align.py to iterate over all entity types — apply intra-source dedup and cross-source alignment to schemas (by property structure), values (by label + value type), and valuesets (by member overlap ≥80%)
+- [X] T041 [US4] Implement valueset alignment heuristic in library/src/undata_library/align.py — compute Jaccard similarity of member sha256 sets between valuesets; merge if overlap ≥ 80%
+- [X] T042 [US4] Implement schema alignment in library/src/undata_library/align.py — compare schemas by their properties list (set of property sha256 hashes); merge if property sets match
+- [X] T043 [US4] Integration test: run all 8 sources → align → verify "Male"/"Female" values from BIDS+NDA+openMINDS are merged; verify sex/gender valuesets are merged; verify alignment report shows per-type stats
 
 ---