Fix concept uniqueness

Concepts were designed to be unique with respect to a lower-cased name,
but that was not consistently enforced in the code, and it prevented
looking up the "equivalent" concept by name. The "cached" normalized
name enables that and the uniqueness constraint works with this new
field.

Author: jberci

web/concepts/models.py

@@ -1,19 +1,20 @@
 import logging
 
-from concepts.utils import UnionFind
+from concepts.utils import UnionFind, normalize_concept_name
 from django.db import models
 from django.db.models.functions import Lower
 from django.db.utils import IntegrityError
 
 
 class Concept(models.Model):
     name = models.CharField(max_length=200, null=True)
+    normal_name = models.CharField(max_length=200, null=True)
     description = models.TextField(null=True)
 
     class Meta:
         ordering = ["name", "description"]
         constraints = [
-            models.UniqueConstraint(Lower("name").desc(), name="unique_lower_name")
+            models.UniqueConstraint(fields=["normal_name"], name="unique_normal_name")
         ]
 
 
@@ -34,7 +35,7 @@ def create_singleton_concepts(self):
                     logging.WARNING,
                     f" A concept named '{new_concept.name}' already exists.",
                 )
-                new_concept = Concept.objects.get(name__iexact=new_concept.name)
+                new_concept = Concept.objects.get(normal_name=new_concept.normal_name)
             item.concept = new_concept
             item.save()
 
@@ -45,16 +46,17 @@ def take_first(lst):
         components = UnionFind(self.all(), Link.objects.all().to_tuples())
         for concept_items in components.get_item_components(sort_key=Item.Source.key()):
             name = take_first([item.name for item in concept_items])
+            normal_name = normalize_concept_name(name)
             description = take_first([item.description for item in concept_items])
-            new_concept = Concept(name=name, description=description)
+            new_concept = Concept(name=name, normal_name=normal_name, description=description)
             try:
                 new_concept.save()
             except IntegrityError:
                 logging.log(
                     logging.WARNING,
                     f" A concept named '{new_concept.name}' already exists.",
                 )
-                new_concept = Concept.objects.get(name=name)
+                new_concept = Concept.objects.get(normal_name=normal_name)
             for item in concept_items:
                 item.concept = new_concept
                 item.save()
@@ -127,7 +129,7 @@ def get_linked_item_urls(self):
         return [i.get_url() for i in self.get_linked_items()]
 
     def to_concept(self):
-        return Concept(name=self.name, description=self.description)
+        return Concept(name=self.name, normal_name=normalize_concept_name(self.name), description=self.description)
 
     def __str__(self):
         if self.name:

web/concepts/utils.py

@@ -1,5 +1,7 @@
 from typing import Any, Dict, List, Tuple
 
+from unidecode import unidecode
+
 
 class UnionFind:
     item_to_element: Dict[Any, int] = {}
@@ -57,3 +59,8 @@ def elements_to_sorted_items(elements: List[int]):
             return items
 
         return list(map(elements_to_sorted_items, self.components.values()))
+
+
+def normalize_concept_name(name: str) -> str:
+    nyoo = unidecode(name).replace('_', '-').replace(' ', '-').lower()
+    return ''.join(filter(lambda x: x.isalnum() or x == '-', nyoo))

web/requirements.txt

@@ -4,6 +4,7 @@ requests~=2.32.5
 spacy~=3.7.0 --prefer-binary
 scispacy~=0.6.2
 python-decouple~=3.8
+unidecode~=1.4.0
 
 # LLM dependencies (optional, install based on which LLM you want to use)
 # For paid APIs: