Recommendations changes (#1203)

Author: brian-kim31

app/airflow/dags/libs/enums.py

@@ -17,4 +17,5 @@ class JobStageType(StrEnum):
     BUILD_CONCEPTS_FROM_DICT = "Build concepts from OMOP Data dictionary"
     REUSE_CONCEPTS = "Reuse concepts from other scan reports"
     GENERATE_RULES = "Generate mapping rules from available concepts"
+    GENERATE_RECOMMENDATIONS = "Generate mapping recommendations"
     DOWNLOAD_RULES = "Generate and download mapping rules JSON"

app/api/api/serializers.py

@@ -23,6 +23,7 @@
     VisibilityChoices,
     UploadStatus,
     MappingStatus,
+    MappingRecommendation,
 )
 from users.serializers import UserSerializer
 from mapping.permissions import has_editorship, is_admin, is_az_function_user
@@ -793,8 +794,26 @@ class Meta:
         fields = ["id", "object_id", "creation_type", "content_type", "concept"]
 
 
+class MappingRecommendationSerializerV3(serializers.ModelSerializer):
+    concept = ConceptSerializerV2()
+
+    class Meta:
+        model = MappingRecommendation
+        fields = [
+            "id",
+            "concept",
+            "score",
+            "tool_name",
+            "tool_version",
+            "created_at",
+        ]
+
+
 class ScanReportValueViewSerializerV3(serializers.ModelSerializer):
     concepts = ScanReportConceptSerializerV2(many=True, read_only=True)
+    mapping_recommendations = MappingRecommendationSerializerV3(
+        many=True, read_only=True
+    )
 
     class Meta:
         model = ScanReportValue
@@ -805,6 +824,7 @@ class Meta:
             "value_description",
             "scan_report_field",
             "concepts",
+            "mapping_recommendations",
         ]
 
 

app/api/api/views.py

@@ -933,7 +933,12 @@ def get_queryset(self):
             ScanReportValue.objects.filter(scan_report_field=self.field)
             .order_by("id")
             .select_related("scan_report_field")
-            .prefetch_related("concepts", "concepts__concept")
+            .prefetch_related(
+                "concepts",
+                "concepts__concept",
+                "mapping_recommendations",
+                "mapping_recommendations__concept",
+            )
         )
 
     def list(self, request, *args, **kwargs):

app/api/test/test_serializers.py

@@ -11,6 +11,19 @@
     ScanReport,
     VisibilityChoices,
 )
+from django.contrib.contenttypes.models import ContentType
+from data.models import Concept
+from mapping.models import (
+    ScanReport,
+    ScanReportTable,
+    ScanReportField,
+    ScanReportValue,
+    MappingRecommendation,
+)
+from api.serializers import (
+    MappingRecommendationSerializerV3,
+    ScanReportValueViewSerializerV3,
+)
 
 
 class TestScanReportEditSerializer(TestCase):
@@ -370,3 +383,87 @@ def test_validate_admin(self):
         # check admin can alter admins
         request.user = self.admin_user
         self.assertEqual(serializer.validate_admins(new_admin), new_admin)
+
+
+class TestMappingRecommendationSerializerV3(TestCase):
+    def setUp(self):
+        # Create test data
+        self.scan_report = ScanReport.objects.create(
+            dataset="Test Dataset",
+            visibility="PUBLIC",
+        )
+
+        self.table = ScanReportTable.objects.create(
+            scan_report=self.scan_report,
+            name="Test Table",
+        )
+
+        self.field = ScanReportField.objects.create(
+            scan_report_table=self.table,
+            name="Test Field",
+            description_column="Test Description",
+            type_column="string",
+        )
+
+        self.value = ScanReportValue.objects.create(
+            scan_report_field=self.field,
+            value="test_value",
+            frequency=5,
+        )
+
+        # Create a concept for the recommendation
+        self.concept = Concept.objects.create(
+            concept_id=12345,
+            concept_name="Test Concept",
+            concept_code="TEST123",
+            domain_id="Test",
+            vocabulary_id="Test",
+            concept_class_id="Test",
+            standard_concept="S",
+            valid_start_date="2020-01-01",
+            valid_end_date="2099-12-31",
+        )
+
+        # Create a mapping recommendation
+        content_type = ContentType.objects.get_for_model(ScanReportValue)
+        self.recommendation = MappingRecommendation.objects.create(
+            content_type=content_type,
+            object_id=self.value.id,
+            concept=self.concept,
+            score=0.85,
+            tool_name="test-tool",
+            tool_version="1.0.0",
+        )
+
+    def test_mapping_recommendation_serializer(self):
+        """Test that MappingRecommendationSerializerV3 serializes correctly."""
+        serializer = MappingRecommendationSerializerV3(self.recommendation)
+        data = serializer.data
+
+        self.assertEqual(data["id"], self.recommendation.id)
+        self.assertEqual(data["score"], 0.85)
+        self.assertEqual(data["tool_name"], "test-tool")
+        self.assertEqual(data["tool_version"], "1.0.0")
+        self.assertEqual(data["concept"]["concept_id"], self.concept.concept_id)
+        self.assertEqual(data["concept"]["concept_name"], self.concept.concept_name)
+
+    def test_scan_report_value_v3_serializer_includes_recommendations(self):
+        """Test that ScanReportValueViewSerializerV3 includes mapping recommendations."""
+        serializer = ScanReportValueViewSerializerV3(self.value)
+        data = serializer.data
+
+        # Check that mapping recommendations are included
+        self.assertIn("mapping_recommendations", data)
+        self.assertEqual(len(data["mapping_recommendations"]), 1)
+
+        recommendation_data = data["mapping_recommendations"][0]
+        self.assertEqual(recommendation_data["id"], self.recommendation.id)
+        self.assertEqual(recommendation_data["score"], 0.85)
+        self.assertEqual(recommendation_data["tool_name"], "test-tool")
+        self.assertEqual(recommendation_data["tool_version"], "1.0.0")
+        self.assertEqual(
+            recommendation_data["concept"]["concept_id"], self.concept.concept_id
+        )
+        self.assertEqual(
+            recommendation_data["concept"]["concept_name"], self.concept.concept_name
+        )

app/api/test/test_views.py

@@ -20,6 +20,7 @@
     ScanReportTable,
     ScanReportValue,
     VisibilityChoices,
+    MappingRecommendation,
 )
 
 
@@ -673,13 +674,11 @@ def setUp(self):
             dataset="The Heights of Hobbits",
             visibility=VisibilityChoices.PUBLIC,
             parent_dataset=self.public_dataset,
-            status="COMPLET",
         )
         self.scanreport2 = ScanReport.objects.create(
             dataset="The Kinds of Orcs",
             visibility=VisibilityChoices.RESTRICTED,
             parent_dataset=self.restricted_dataset,
-            status="COMPLET",
         )
 
         # Set up projects
@@ -701,12 +700,6 @@ def setUp(self):
             name="Field1",
             description_column="",
             type_column="",
-            max_length=32,
-            nrows=0,
-            nrows_checked=0,
-            fraction_empty=0.0,
-            nunique_values=0,
-            fraction_unique=0.0,
         )
         self.scanreportvalue1 = ScanReportValue.objects.create(
             scan_report_field=self.scanreportfield1,
@@ -745,12 +738,6 @@ def setUp(self):
             name="Field2",
             description_column="",
             type_column="",
-            max_length=32,
-            nrows=0,
-            nrows_checked=0,
-            fraction_empty=0.0,
-            nunique_values=0,
-            fraction_unique=0.0,
         )
         self.scanreportvalue2 = ScanReportValue.objects.create(
             scan_report_field=self.scanreportfield2,
@@ -805,3 +792,82 @@ def test_az_function_user_get(self):
         az_response_ids = [item["id"] for item in az_response.data]
         self.assertTrue(self.scanreportconcept2.id in az_response_ids)
         self.assertTrue(self.scanreportconcept4.id in az_response_ids)
+
+    def test_scan_report_value_list_v3_includes_recommendations(self):
+        """Test that ScanReportValueListV3 includes mapping recommendations."""
+        # Create test data
+        scan_report = ScanReport.objects.create(
+            dataset="Test Dataset",
+            visibility=VisibilityChoices.PUBLIC,
+            parent_dataset=self.public_dataset,
+        )
+
+        table = ScanReportTable.objects.create(
+            scan_report=scan_report,
+            name="Test Table",
+        )
+
+        field = ScanReportField.objects.create(
+            scan_report_table=table,
+            name="Test Field",
+            description_column="Test Description",
+            type_column="string",
+        )
+
+        value = ScanReportValue.objects.create(
+            scan_report_field=field,
+            value="test_value",
+            frequency=5,
+        )
+
+        # Create a concept for the recommendation
+        concept = Concept.objects.create(
+            concept_id=12345,
+            concept_name="Test Concept",
+            concept_code="TEST123",
+            domain_id="Test",
+            vocabulary_id="Test",
+            concept_class_id="Test",
+            standard_concept="S",
+            valid_start_date="2020-01-01",
+            valid_end_date="2099-12-31",
+        )
+
+        # Create a mapping recommendation
+        content_type = ContentType.objects.get_for_model(ScanReportValue)
+        recommendation = MappingRecommendation.objects.create(
+            content_type=content_type,
+            object_id=value.id,
+            concept=concept,
+            score=0.85,
+            tool_name="test-tool",
+            tool_version="1.0.0",
+        )
+
+        # Make request
+        url = f"/api/v3/scanreports/{scan_report.id}/tables/{table.id}/fields/{field.id}/values/"
+        response = self.client.get(url)
+
+        # Verify response
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+
+        # Check that the value is returned
+        self.assertEqual(len(data["results"]), 1)
+        value_data = data["results"][0]
+
+        # Check that mapping recommendations are included
+        self.assertIn("mapping_recommendations", value_data)
+        self.assertEqual(len(value_data["mapping_recommendations"]), 1)
+
+        recommendation_data = value_data["mapping_recommendations"][0]
+        self.assertEqual(recommendation_data["id"], recommendation.id)
+        self.assertEqual(recommendation_data["score"], 0.85)
+        self.assertEqual(recommendation_data["tool_name"], "test-tool")
+        self.assertEqual(recommendation_data["tool_version"], "1.0.0")
+        self.assertEqual(
+            recommendation_data["concept"]["concept_id"], concept.concept_id
+        )
+        self.assertEqual(
+            recommendation_data["concept"]["concept_name"], concept.concept_name
+        )

app/next-client-app/api/recommendations.ts

@@ -6,7 +6,6 @@ import {
   unisonApiKey,
   recommendationService,
 } from "@/constants";
-import { UnisonConceptResponse } from "@/types/recommendation";
 
 export const getConceptRecommendationsUnison = async (
   // Unison can query by concept name, or concept code (exact match).

app/next-client-app/app/(protected)/datasets/[id]/page.tsx

@@ -4,7 +4,6 @@ import { DataTable } from "@/components/data-table";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { objToQuery } from "@/lib/client-utils";
 import { ScanReportsTableFilter } from "@/components/scanreports/ScanReportsTableFilter";
-import { FilterParameters } from "@/types/filter";
 import { VisibilityState } from "@tanstack/react-table";
 import { EmptyState } from "@/components/ui/empty-state";
 

app/next-client-app/app/(protected)/datasets/page.tsx

@@ -4,7 +4,6 @@ import { getDataPartners, getDataSets } from "@/api/datasets";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { objToQuery } from "@/lib/client-utils";
 import { DataTableFilter } from "@/components/data-table/DataTableFilter";
-import { FilterParameters } from "@/types/filter";
 import { CreateDatasetDialog } from "@/components/datasets/CreateDatasetDialog";
 import { Database } from "lucide-react";
 import { getAllProjects } from "@/api/projects";

app/next-client-app/app/(protected)/projects/[id]/page.tsx

@@ -1,6 +1,5 @@
 import { DataTable } from "@/components/data-table";
 import { objToQuery } from "@/lib/client-utils";
-import { FilterParameters } from "@/types/filter";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { DataTableFilter } from "@/components/data-table/DataTableFilter";
 import { getDataSets } from "@/api/datasets";

app/next-client-app/app/(protected)/projects/page.tsx

@@ -2,7 +2,6 @@ import { DataTable } from "@/components/data-table";
 import { columns } from "./columns";
 import { objToQuery } from "@/lib/client-utils";
 import { DataTableFilter } from "@/components/data-table/DataTableFilter";
-import { FilterParameters } from "@/types/filter";
 import { Folders } from "lucide-react";
 import { getProjectsList } from "@/api/projects";
 import { Metadata } from "next";