Remove code that creates NEXT relationships between facts (#267)

Co-authored-by: ianrob <ianrob@amazon.co.uk>

Author: Ian Robinson

docs-site/src/content/docs/lexical-graph/graph-model.mdx

@@ -74,8 +74,6 @@ There are two types of fact: subject-predicate-object (SPO) facts, and subject-p
 Neptune Analytics PURPOSE analyze graph data
 ```
 
-SPO facts are connected to other facts via `__NEXT__` relationships, where the object entity of a first fact acts as the subject entity for a subsequent fact.
-
 Facts provide *connectivity across different sources*. It's not uncommon for an individual fact to be mentioned multiple times in the underlying corpus: for example, in a news articles dataset, a particular fact might be repeated in different news articles reporting on the same story. In the graph, there will be a single node to represent this specific fact. From this node it is then possible to traverse via statements, topics and chunks to all the places where that particular fact is mentioned.
 
 Every fact `__SUPPORTS__` at least one statement. A fact can support multiple statements, belonging to the same or different topics and sources.

examples/lexical-graph/scripts/repair_facts.py

@@ -290,47 +290,6 @@ def create_entity_entity_relation(graph_store, facts):
     graph_store.execute_query_with_retry(cypher, parameters)
 
 
-    
-def create_fact_next_relation(graph_store, facts):
-
-    params = []
-    
-    for fact in facts:
-        params.append({
-            'fact_id': fact['fact_id']
-        })
-    
-    parameters = {
-        'params': params    
-    }
-    
-    
-    statements_prev = [
-        '// insert connection to prev facts',
-        'UNWIND $params AS params',
-        f'MATCH (fact:`__Fact__`{{{graph_store.node_id("factId")}: params.fact_id}})<-[:`__SUBJECT__`]-(:`__Entity__`)-[:`__OBJECT__`]->(prevFact:`__Fact__`)',
-        'WHERE fact <> prevFact // and NOT ((fact)<-[:`__NEXT__`]-(prevFact))',
-        'WITH DISTINCT fact, prevFact',
-        'MERGE (fact)<-[:`__NEXT__`]-(prevFact)'
-    ]
-
-    query_prev = '\n'.join(statements_prev)
-        
-    graph_store.execute_query_with_retry(query_prev, parameters, max_attempts=5, max_wait=7)
-    
-    statements_next = [
-        '// insert connection to next facts',
-        'UNWIND $params AS params',
-        f'MATCH (fact:`__Fact__`{{{graph_store.node_id("factId")}: params.fact_id}})<-[:`__OBJECT__`]-(:`__Entity__`)-[:`__SUBJECT__`]->(nextFact:`__Fact__`)',
-        'WHERE fact <> nextFact // and NOT ((fact)-[:`__NEXT__`]->(nextFact))',
-        'WITH DISTINCT fact, nextFact',
-        'MERGE (fact)-[:`__NEXT__`]->(nextFact)'
-    ]
-
-    query_next = '\n'.join(statements_next)
-        
-    graph_store.execute_query_with_retry(query_next, parameters, max_attempts=5, max_wait=7)
-    
 def get_stats(graph_store, fact_ids, batch_size):
 
     stats = {}
@@ -377,57 +336,6 @@ def get_stats(graph_store, fact_ids, batch_size):
 
     stats['num_object_relationships'] = total_object
 
-    
-    
-
-    #cypher = '''
-    #MATCH (:`__Entity__`)-[r:`__SUBJECT__`]->()
-    #RETURN count(r) AS count
-    #'''
-    #
-    #results = graph_store.execute_query_with_retry(cypher, {})
-    #
-    #stats['num_subject_relationships'] = results[0]['count']
-    #
-    #cypher = '''
-    #MATCH (:`__Entity__`)-[r:`__OBJECT__`]->()
-    #RETURN count(r) AS count
-    #'''
-    #
-    #results = graph_store.execute_query_with_retry(cypher, {})
-    #
-    #stats['num_object_relationships'] = results[0]['count']
-    
-    #cypher = '''
-    #MATCH (:`__Entity__`)-[r:`__RELATION__`]->(:`__Entity__`)
-    #RETURN count(r) AS count
-    #'''
-    #
-    #results = graph_store.execute_query_with_retry(cypher, {})
-    #
-    #stats['num_relation_relationships'] = results[0]['count']
-    
-    total_next = 0
-
-    progress_bar_1 = tqdm(total=len(fact_ids), desc='Counting NEXT relationships')
-    for fact_id_batch in iter_batch(fact_ids, batch_size=batch_size):
-        cypher = '''
-        MATCH (f)-[r:`__NEXT__`]->() WHERE id(f) in $fact_ids
-        RETURN count(r) AS count
-        '''
-    
-        params = {
-            'fact_ids': fact_id_batch
-        }
-    
-        results = graph_store.execute_query_with_retry(cypher, params)
-    
-        counts = [r['count'] for r in results]
-        total_next += sum(counts)
-        progress_bar_1.update(len(fact_id_batch))
-    
-    stats['num_next_relationships'] = total_next
-    
     return stats
 
 def iter_batch(iterable, batch_size):
@@ -477,23 +385,6 @@ def repair(graph_store_info, batch_size, skip_invalid_relationships, skip_entity
             create_entity_fact_relation(graph_store, facts, 'object')
             progress_bar_1.update(len(fact_id_batch))
 
-    #print()
-    #print('Creating RELATION entity-entity relationships...')
-    #total = 0
-    #for fact_id_batch in iter_batch(fact_ids, batch_size=batch_size):
-    #    facts = get_facts(graph_store, fact_id_batch)
-    #    create_entity_entity_relation(graph_store, facts)
-    #    total += len(fact_id_batch)
-    #    if total % TOTAL_MOD == 0:
-    #        print(f'  {total}')
-    #print(f'  Done')
-
-    progress_bar_2 = tqdm(total=len(fact_ids_to_process), desc='Creating NEXT fact-fact relationships')
-    for fact_id_batch in iter_batch(fact_ids_to_process, batch_size=batch_size):    
-        facts = get_facts(graph_store, fact_id_batch)
-        create_fact_next_relation(graph_store, facts)
-        progress_bar_2.update(len(fact_id_batch))
-
     stats['after'] = get_stats(graph_store, fact_ids, batch_size)
 
     return stats

images/lexical-graph.png

@@ -61,12 +61,5 @@ def test_contains_correct_number_of_facts(self):
 
                     self.assertEqual(4, fact_node_count)
 
-                def test_contains_correct_number_of_next_relationships(self):
-                    """Graph contains correct number of next relationships"""
-                    
-                    results = self._graph_store.execute_query('MATCH (:`__Fact__`)-[r:`__NEXT__`]->() RETURN count(r) AS count')
-                    next_relationship_count = results[0]['count']
-                    
-                    self.assertEqual(3, next_relationship_count)
 
             handler.run_assertions(BuildFactsAssertions)

integration-tests/test-scripts/graphrag_toolkit_tests/build_facts.py

@@ -118,13 +118,6 @@ def test_fact_supports_statement_relationships(self):
 
                 self.assertEqual(relationship_count, 43)
 
-            def test_fact_next_fact_relationships(self):
-                """Graph contains expected fact NEXT fact relationships"""
-                
-                results = self._graph_store.execute_query('MATCH (:`__Fact__`)<-[r:`__NEXT__`]-(:`__Fact__`) RETURN count(r) AS count')
-                relationship_count = results[0]['count']
-                
-                self.assertEqual(relationship_count, 27)
 
             def test_entity_subject_fact_relationships(self):
                 """Graph contains expected entity SUBJECT fact relationships"""
@@ -250,13 +243,6 @@ def test_fact_supports_statement_relationships(self):
 
                 self.assertEqual(relationship_count, 43)
 
-            def test_fact_next_fact_relationships(self):
-                """Graph contains expected fact NEXT fact relationships"""
-                
-                results = self._graph_store.execute_query('MATCH (:`__Fact__`)<-[r:`__NEXT__`]-(:`__Fact__`) RETURN count(r) AS count')
-                relationship_count = results[0]['count']
-                
-                self.assertEqual(relationship_count, 15)
 
             def test_entity_subject_fact_relationships(self):
                 """Graph contains expected entity SUBJECT fact relationships"""

integration-tests/test-scripts/graphrag_toolkit_tests/local_entities.py

@@ -129,52 +129,6 @@ def insert_entity_fact_relationship(relationship_type:str, entity_id:Optional[st
                     insert_entity_fact_relationship('object', fact.object.entityId)          
                 if fact.complement and include_local_entities:
                     insert_entity_fact_relationship('object', fact.complement.entityId)
-
-            create_next_relationship = Query(
-                query=f"""// insert next connection between facts
-                UNWIND $params AS params
-                MATCH (start{{{graph_client.node_id("factId")}: params.startId}}), (end{{{graph_client.node_id("factId")}: params.endId}})
-                MERGE (start)-[:`__NEXT__`]->(end)
-                """
-            )
-
-            find_start_end_for_prev_facts = Query(
-                query=f"""// get start and end facts for prev connection
-                UNWIND $params AS params
-                MATCH (fact:`__Fact__`{{{graph_client.node_id("factId")}: params.fact_id}})<-[:`__SUBJECT__`]-(:`__Entity__`)-[:`__OBJECT__`]->(prevFact:`__Fact__`)
-                WHERE fact <> prevFact
-                RETURN {graph_client.node_id('prevFact.factId')} AS startId, {graph_client.node_id('fact.factId')} AS endId
-                """,
-                child_queries=[create_next_relationship]
-            )
-
-            params = {
-                'fact_id': fact.factId
-            }
-
-            query_tree = QueryTree('insert-prev-facts', find_start_end_for_prev_facts)
-
-            graph_client.execute_query_with_retry(query_tree, self._to_params(params), max_attempts=10, max_wait=10)
-
-            if fact.object or fact.complement:
-
-                find_start_end_for_next_facts = Query(
-                    query=f"""// get start and end facts for next connection
-                    UNWIND $params AS params
-                    MATCH (fact:`__Fact__`{{{graph_client.node_id("factId")}: params.fact_id}})<-[:`__OBJECT__`]-(:`__Entity__`)-[:`__SUBJECT__`]->(nextFact:`__Fact__`)
-                    WHERE fact <> nextFact
-                    RETURN {graph_client.node_id('fact.factId')} AS startId, {graph_client.node_id('nextFact.factId')} AS endId
-                    """,
-                    child_queries=[create_next_relationship]
-                )
-
-                params = {
-                    'fact_id': fact.factId
-                }
-
-                query_tree = QueryTree('insert-next-facts', find_start_end_for_next_facts)
-
-                graph_client.execute_query_with_retry(query_tree, self._to_params(params), max_attempts=10, max_wait=10)
 
         else:
             logger.warning(f'fact_id missing from fact node [node_id: {node.node_id}]')

lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/fact_graph_builder.py

@@ -190,7 +190,7 @@ def _single_entity_based_graph_search(self, entity_id, query:QueryBundle):
 
         cypher = f'''// single entity-based graph search                            
         MATCH (:`__Entity__`{{{self.graph_store.node_id("entityId")}:$startId}})
-            -[:`__SUBJECT__`]->()-[:`__NEXT__`*0..1]-()
+            -[:`__SUBJECT__`]->()
             -[:`__SUPPORTS__`]->()
             -[:`__PREVIOUS__`*0..1]-(l)
         RETURN DISTINCT {self.graph_store.node_id("l.statementId")} AS l LIMIT $statementLimit'''

lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_based_search.py

@@ -80,7 +80,7 @@ def topic_based_graph_search(self, topic_id):
         query to traverse a graph database, retrieving relevant `__Fact__` and associated
         `__Statement__` nodes connected to a specific topic.
 
-        The query traverses relationships such as `__NEXT__`, `__SUPPORTS__`, and
+        The query traverses relationships such as `__SUPPORTS__`, and
         `__BELONGS_TO__` to ensure that all relevant nodes and their connections are
         retrieved based on the provided topic ID and query limits.
 
@@ -99,7 +99,7 @@ def topic_based_graph_search(self, topic_id):
         MATCH (f)-[:`__SUPPORTS__`]->()-[:`__BELONGS_TO__`]->(tt:`__Topic__`)
         WHERE {self.graph_store.node_id("tt.topicId")} = $topicId
         WITH f LIMIT $statementLimit
-        MATCH (f)-[:`__NEXT__`*0..1]-()-[:`__SUPPORTS__`]->(l)
+        MATCH (f)-[:`__SUPPORTS__`]->(l)
         RETURN DISTINCT {self.graph_store.node_id("l.statementId")} AS l LIMIT $statementLimit
         '''