Merge pull request #43 from alphagov/ACW-70/similarity-distance

Calculating textual/edit distance between aliases

Author: daymohlar

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "requests-aws4auth>=1.3.1",
     "flask[async]>=3.1.3",
     "uvicorn>=0.30.0",
+    "levenshtein>=0.27.3",
 ]
 
 [dependency-groups]

src/models/graph_models.py

@@ -62,7 +62,26 @@ class Edge(BaseModel):
     data: EdgeData
 
 
+class SimilarAlias(BaseModel):
+    id: str
+    label: str
+    similarity: int
+
+
+class OutlierAlias(BaseModel):
+    id: str
+    label: str
+    similar_aliases: List[SimilarAlias] = Field(default_factory=list)
+
+
+class EntityOutlier(BaseModel):
+    entity_id: str
+    entity_label: str
+    aliases: List[OutlierAlias] = Field(default_factory=list)
+
+
 class GraphOutput(BaseModel):
     nodes: List[Node]
     edges: List[Edge]
     relationships: List[Relationship] = Field(default_factory=list)
+    outliers: List[EntityOutlier] = Field(default_factory=list)

src/utils/alias_distance.py

@@ -0,0 +1,6 @@
+from Levenshtein import distance
+
+
+def calculate_alias_distance(alias1: str, alias2: str) -> int:
+    """Calculate the Levenshtein distance between two aliases."""
+    return distance(alias1, alias2)

src/visualiser_graph_generator.py

@@ -15,13 +15,17 @@
     Edge,
     EdgeData,
     Entity,
+    EntityOutlier,
     GraphInput,
     GraphOutput,
     Node,
     NodeData,
     Occurrence,
+    OutlierAlias,
     Relationship,
+    SimilarAlias,
 )
+from src.utils.alias_distance import calculate_alias_distance
 
 
 logger = logging.getLogger(__name__)
@@ -181,6 +185,7 @@ def build_node_structure(
 ) -> GraphOutput:
     """Constructs the final list of nodes and edges."""
     nodes, edges = [], []
+    outliers = []
     id_to_canonical = {ent.id: ent.canonical_key for ent in entities}
 
     for ent in entities:
@@ -205,7 +210,9 @@ def build_node_structure(
                 occ.extend(occurrences)
 
         # Add the deduplicated alias nodes and their edges
-        for alias_id, node_data in alias_map.items():
+        unique_aliases = list(alias_map.values())
+        for node_data in unique_aliases:
+            alias_id = node_data.id
             if not node_data.occurrences:
                 node_data.occurrences = None
 
@@ -223,6 +230,36 @@ def build_node_structure(
                 )
             )
 
+        # Build outlier aliases structure for this entity
+        outlier_aliases = []
+        for curr_alias in unique_aliases:
+            similar_aliases = []
+            for other_alias in unique_aliases:
+                if other_alias.id != curr_alias.id:
+                    dist = calculate_alias_distance(curr_alias.label, other_alias.label)
+                    similar_aliases.append(
+                        SimilarAlias(
+                            id=other_alias.id,
+                            label=other_alias.label,
+                            similarity=dist,
+                        )
+                    )
+            outlier_aliases.append(
+                OutlierAlias(
+                    id=curr_alias.id,
+                    label=curr_alias.label,
+                    similar_aliases=similar_aliases,
+                )
+            )
+
+        outliers.append(
+            EntityOutlier(
+                entity_id=ent_id,
+                entity_label=human_label,
+                aliases=outlier_aliases,
+            )
+        )
+
     for rel in relationships or []:
         source = id_to_canonical.get(rel.from_, rel.from_)
         target = id_to_canonical.get(rel.to, rel.to)
@@ -237,7 +274,12 @@ def build_node_structure(
             )
         )
 
-    return GraphOutput(nodes=nodes, edges=edges, relationships=relationships or [])
+    return GraphOutput(
+        nodes=nodes,
+        edges=edges,
+        relationships=relationships or [],
+        outliers=outliers,
+    )
 
 
 async def generate_graph(