diff --git a/python/test_utils/filters_setup.py b/python/test_utils/filters_setup.py index 6784785aac..610413e71c 100644 --- a/python/test_utils/filters_setup.py +++ b/python/test_utils/filters_setup.py @@ -284,3 +284,88 @@ def init_edges_graph2(graph): graph.edge(src, dst).add_constant_properties(props) return graph + + +import tempfile +from raphtory.graphql import GraphServer +import json +import re + +PORT = 1737 + + +def create_test_graph(g): + g.add_node( + 1, + "a", + { + "prop1": 60, + "prop2": 31.3, + "prop3": "abc123", + "prop4": True, + "prop5": [1, 2, 3], + }, + "fire_nation" + ) + g.add_node( + 1, + "b", + {"prop1": 10, "prop2": 31.3, "prop3": "abc223", "prop4": False}, + "fire_nation" + ) + g.add_node( + 1, + "c", + { + "prop1": 20, + "prop2": 31.3, + "prop3": "abc333", + "prop4": True, + "prop5": [5, 6, 7], + }, + "water_tribe" + ) + g.add_node( + 1, + "d", + {"prop1": 30, "prop2": 31.3, "prop3": "abc444", "prop4": False}, + "air_nomads" + ) + g.add_edge( + 2, + "a", + "d", + { + "eprop1": 60, + "eprop2": 0.4, + "eprop3": "xyz123", + "eprop4": True, + "eprop5": [1, 2, 3], + }, + ) + g.add_edge( + 2, + "b", + "d", + { + "eprop1": 10, + "eprop2": 1.7, + "eprop3": "xyz123", + "eprop4": True, + "eprop5": [3, 4, 5], + }, + ) + g.add_edge( + 2, + "c", + "d", + { + "eprop1": 30, + "eprop2": 6.4, + "eprop3": "xyz123", + "eprop4": False, + "eprop5": [10], + }, + ) + return g + diff --git a/python/test_utils/utils.py b/python/test_utils/utils.py index aa725d224b..a950b5530f 100644 --- a/python/test_utils/utils.py +++ b/python/test_utils/utils.py @@ -135,6 +135,24 @@ def run_graphql_error_test(query, expected_error_message, graph): ), f"Expected '{expected_error_message}', but got '{error_message}'" +def run_graphql_error_test(query, expected_error_message, graph): + tmp_work_dir = tempfile.mkdtemp() + with GraphServer(tmp_work_dir).start(PORT) as server: + client = server.get_client() + client.send_graph(path="g", graph=graph) + + with pytest.raises(Exception) as excinfo: + client.query(query) + + full_error_message = str(excinfo.value) + match = re.search(r'"message":"(.*?)"', full_error_message) + error_message = match.group(1) if match else "" + + assert ( + error_message == expected_error_message + ), f"Expected '{expected_error_message}', but got '{error_message}'" + + def run_group_graphql_error_test(queries_and_expected_error_messages, graph): tmp_work_dir = tempfile.mkdtemp() with GraphServer(tmp_work_dir).start(PORT) as server: diff --git a/python/tests/test_base_install/test_graphql/test_edge_sorting.py b/python/tests/test_base_install/test_graphql/test_edge_sorting.py index 396f09608a..4b7907e7f6 100644 --- a/python/tests/test_base_install/test_graphql/test_edge_sorting.py +++ b/python/tests/test_base_install/test_graphql/test_edge_sorting.py @@ -1,7 +1,5 @@ import pytest - from raphtory import Graph, PersistentGraph - from utils import run_graphql_test @@ -67,12 +65,7 @@ def create_test_graph(g): return g -EVENT_GRAPH = create_test_graph(Graph()) - -PERSISTENT_GRAPH = create_test_graph(PersistentGraph()) - - -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_nothing(graph): query = """ query { @@ -107,10 +100,11 @@ def test_graph_edge_sort_by_nothing(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_src(graph): query = """ query { @@ -145,10 +139,11 @@ def test_graph_edge_sort_by_src(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_dst(graph): query = """ query { @@ -183,10 +178,11 @@ def test_graph_edge_sort_by_dst(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_earliest_time(graph): query = """ query { @@ -221,10 +217,11 @@ def test_graph_edge_sort_by_earliest_time(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_earliest_time_reversed(graph): query = """ query { @@ -260,10 +257,11 @@ def test_graph_edge_sort_by_earliest_time_reversed(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph]) def test_graph_edge_sort_by_latest_time(graph): query = """ query { @@ -298,10 +296,11 @@ def test_graph_edge_sort_by_latest_time(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [PersistentGraph]) def test_graph_edge_sort_by_latest_time_persistent_graph(graph): query = """ query { @@ -337,10 +336,11 @@ def test_graph_edge_sort_by_latest_time_persistent_graph(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_eprop1(graph): query = """ query { @@ -375,10 +375,11 @@ def test_graph_edge_sort_by_eprop1(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_eprop2(graph): query = """ query { @@ -413,10 +414,11 @@ def test_graph_edge_sort_by_eprop2(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_eprop3(graph): query = """ query { @@ -451,10 +453,11 @@ def test_graph_edge_sort_by_eprop3(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_eprop4(graph): query = """ query { @@ -489,10 +492,11 @@ def test_graph_edge_sort_by_eprop4(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_eprop5(graph): query = """ query { @@ -527,10 +531,11 @@ def test_graph_edge_sort_by_eprop5(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_nonexistent_prop(graph): query = """ query { @@ -565,10 +570,11 @@ def test_graph_edge_sort_by_nonexistent_prop(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_combined(graph): query = """ query { @@ -603,10 +609,11 @@ def test_graph_edge_sort_by_combined(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) -@pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_graph_edge_sort_by_combined_2(graph): query = """ query { @@ -641,4 +648,5 @@ def test_graph_edge_sort_by_combined_2(graph): } } } + graph = create_test_graph(graph()) run_graphql_test(query, expected_output, graph) diff --git a/python/tests/test_base_install/test_graphql/test_graph_nodes_edges_property_filter.py b/python/tests/test_base_install/test_graphql/test_filters/test_graph_edges_property_filter.py similarity index 55% rename from python/tests/test_base_install/test_graphql/test_graph_nodes_edges_property_filter.py rename to python/tests/test_base_install/test_graphql/test_filters/test_graph_edges_property_filter.py index 60c1ec0992..7056650e34 100644 --- a/python/tests/test_base_install/test_graphql/test_graph_nodes_edges_property_filter.py +++ b/python/tests/test_base_install/test_graphql/test_filters/test_graph_edges_property_filter.py @@ -1,929 +1,7 @@ -import tempfile - import pytest - -from raphtory.graphql import GraphServer from raphtory import Graph, PersistentGraph -import json -import re - -PORT = 1737 - - -def create_test_graph(g): - g.add_node( - 1, - "a", - properties={ - "prop1": 60, - "prop2": 31.3, - "prop3": "abc123", - "prop4": True, - "prop5": [1, 2, 3], - }, - ) - g.add_node( - 1, - "b", - properties={"prop1": 10, "prop2": 31.3, "prop3": "abc223", "prop4": False}, - ) - g.add_node( - 1, - "c", - properties={ - "prop1": 20, - "prop2": 31.3, - "prop3": "abc333", - "prop4": True, - "prop5": [5, 6, 7], - }, - ) - g.add_node( - 1, - "d", - properties={"prop1": 30, "prop2": 31.3, "prop3": "abc444", "prop4": False}, - ) - g.add_edge( - 2, - "a", - "d", - properties={ - "eprop1": 60, - "eprop2": 0.4, - "eprop3": "xyz123", - "eprop4": True, - "eprop5": [1, 2, 3], - }, - ) - g.add_edge( - 2, - "b", - "d", - properties={ - "eprop1": 10, - "eprop2": 1.7, - "eprop3": "xyz123", - "eprop4": True, - "eprop5": [3, 4, 5], - }, - ) - g.add_edge( - 2, - "c", - "d", - properties={ - "eprop1": 30, - "eprop2": 6.4, - "eprop3": "xyz123", - "eprop4": False, - "eprop5": [10], - }, - ) - return g - - -def run_graphql_test(query, expected_output, graph): - create_test_graph(graph) - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(PORT) as server: - client = server.get_client() - client.send_graph(path="g", graph=graph) - - response = client.query(query) - - # Convert response to a dictionary if needed and compare - response_dict = json.loads(response) if isinstance(response, str) else response - assert response_dict == expected_output - - -def run_graphql_error_test(query, expected_error_message, graph): - create_test_graph(graph) - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(PORT) as server: - client = server.get_client() - client.send_graph(path="g", graph=graph) - - with pytest.raises(Exception) as excinfo: - client.query(query) - - full_error_message = str(excinfo.value) - match = re.search(r'"message":"(.*?)"', full_error_message) - error_message = match.group(1) if match else "" - - assert ( - error_message == expected_error_message - ), f"Expected '{expected_error_message}', but got '{error_message}'" - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_equal(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop5" - operator: EQUAL - value: { list: [ {i64: 1}, {i64: 2}, {i64: 3} ] } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_equal_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop5" - operator: EQUAL - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for Equal operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_equal_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop5" - operator: EQUAL - value: { i64: 1 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop5: expected List(I64) but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_not_equal(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop4" - operator: NOT_EQUAL - value: { bool: true } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_not_equal_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop4" - operator: NOT_EQUAL - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for NotEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_not_equal_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop4" - operator: NOT_EQUAL - value: { i64: 1 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop4: expected Bool but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than_or_equal(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN_OR_EQUAL - value: { i64: 60 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than_or_equal_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN_OR_EQUAL - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for GreaterThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than_or_equal_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN_OR_EQUAL - value: { bool: true } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Bool" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than_or_equal(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN_OR_EQUAL - value: { i64: 30 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": { - "nodeFilter": { - "nodes": {"list": [{"name": "b"}, {"name": "c"}, {"name": "d"}]} - } - } - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than_or_equal_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN_OR_EQUAL - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for LessThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than_or_equal_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN_OR_EQUAL - value: { str: "shivam" } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN - value: { i64: 30 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for GreaterThan operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_greater_than_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: GREATER_THAN - value: { str: "shivam" } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN - value: { i64: 30 } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "c"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for LessThan operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_less_than_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: LESS_THAN - value: { str: "shivam" } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_none(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop5" - operator: IS_NONE - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_some(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop5" - operator: IS_SOME - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}, {"name": "c"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_in(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_IN - value: { list: [{i64: 10},{i64: 30},{i64: 50},{i64: 70}]} - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_node_property_filter_is_in_empty_list(graph): - query = """ - query { - graph(path: "g") { - nodes { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_IN - value: { list: []} - } - } - ) { - list { - name - } - } - } - } - } - """ - expected_output = {"graph": {"nodes": {"nodeFilter": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_in_no_value(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_IN - value: { list: []} - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = {"graph": {"nodeFilter": {"nodes": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_in_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_IN - value: { str: "shivam" } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_not_in_any(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_NOT_IN - value: { list: [{i64: 10},{i64: 30},{i64: 50},{i64: 70}]} - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}, {"name": "c"}]}}} - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_node_property_filter_not_is_not_in_empty_list(graph): - query = """ - query { - graph(path: "g") { - nodes { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_NOT_IN - value: { list: []} - } - } - ) { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": { - "nodes": { - "nodeFilter": { - "list": [{"name": "a"}, {"name": "b"}, {"name": "c"}, {"name": "d"}] - } - } - } - } - run_graphql_test(query, expected_output, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_not_in_no_value_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_NOT_IN - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "Expected a value for IsNotIn operator" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_property_filter_is_not_in_type_error(graph): - query = """ - query { - graph(path: "g") { - nodeFilter( - filter: { - property: { - name: "prop1" - operator: IS_NOT_IN - value: { str: "shivam" } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) - - -@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) -def test_graph_node_not_property_filter(graph): - query = """ - query { - graph(path: "g") { - nodeFilter ( - filter: { - not: - { - property: { - name: "prop5" - operator: EQUAL - value: { list: [ {i64: 1}, {i64: 2} ] } - } - } - } - ) { - nodes { - list { - name - } - } - } - } - } - """ - expected_output = { - "graph": { - "nodeFilter": { - "nodes": { - "list": [{"name": "a"}, {"name": "b"}, {"name": "c"}, {"name": "d"}] - } - } - } - } - run_graphql_test(query, expected_output, graph()) +from filters_setup import PORT, create_test_graph +from utils import run_graphql_test, run_graphql_error_test # Edge property filter is not supported yet for PersistentGraph @@ -958,7 +36,8 @@ def test_graph_edge_property_filter_equal(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_equal_persistent_graph(): @@ -985,7 +64,8 @@ def test_graph_edge_property_filter_equal_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1012,7 +92,8 @@ def test_graph_edge_property_filter_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for Equal operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1040,7 +121,8 @@ def test_graph_edge_property_filter_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop5: expected List(I64) but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_equal_type_error_persistent_graph(): @@ -1066,7 +148,8 @@ def test_graph_edge_property_filter_equal_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1101,7 +184,8 @@ def test_graph_edge_property_filter_not_equal(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_not_equal_persistent_graph(): @@ -1128,7 +212,8 @@ def test_graph_edge_property_filter_not_equal_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1155,7 +240,8 @@ def test_graph_edge_property_filter_not_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for NotEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1184,7 +270,8 @@ def test_graph_edge_property_filter_not_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop4: expected Bool but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_not_equal_type_error_persistent_graph(): @@ -1211,7 +298,8 @@ def test_graph_edge_property_filter_not_equal_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1246,7 +334,8 @@ def test_graph_edge_property_filter_greater_than_or_equal(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_greater_than_or_equal_persistent_graph(): @@ -1273,7 +362,8 @@ def test_graph_edge_property_filter_greater_than_or_equal_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1300,7 +390,8 @@ def test_graph_edge_property_filter_greater_than_or_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for GreaterThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1329,7 +420,8 @@ def test_graph_edge_property_filter_greater_than_or_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Bool" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_greater_than_or_equal_type_error_persistent_graph(): @@ -1356,7 +448,8 @@ def test_graph_edge_property_filter_greater_than_or_equal_type_error_persistent_ } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1396,7 +489,8 @@ def test_graph_edge_property_filter_less_than_or_equal(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_less_than_or_equal_persistent_graph(): @@ -1423,7 +517,8 @@ def test_graph_edge_property_filter_less_than_or_equal_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1450,7 +545,8 @@ def test_graph_edge_property_filter_less_than_or_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for LessThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1479,7 +575,8 @@ def test_graph_edge_property_filter_less_than_or_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_less_than_or_equal_type_error_persistent_graph(): @@ -1506,7 +603,8 @@ def test_graph_edge_property_filter_less_than_or_equal_type_error_persistent_gra } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1541,7 +639,8 @@ def test_graph_edge_property_filter_greater_than(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_greater_than_persistent_graph(): @@ -1568,7 +667,8 @@ def test_graph_edge_property_filter_greater_than_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1595,7 +695,8 @@ def test_graph_edge_property_filter_greater_than_no_value_error(graph): } """ expected_error_message = "Expected a value for GreaterThan operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1624,7 +725,8 @@ def test_graph_edge_property_filter_greater_than_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_greater_than_type_error_persistent_graph(): @@ -1651,7 +753,8 @@ def test_graph_edge_property_filter_greater_than_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1686,7 +789,8 @@ def test_graph_edge_property_filter_less_than(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_less_than_persistent_graph(): @@ -1713,7 +817,8 @@ def test_graph_edge_property_filter_less_than_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -1740,7 +845,8 @@ def test_graph_edge_property_filter_less_than_no_value_error(graph): } """ expected_error_message = "Expected a value for LessThan operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1769,7 +875,8 @@ def test_graph_edge_property_filter_less_than_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_less_than_type_error_persistent_graph(): @@ -1796,7 +903,8 @@ def test_graph_edge_property_filter_less_than_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1824,7 +932,8 @@ def test_graph_edge_property_filter_is_none(graph): } """ expected_output = {"graph": {"edgeFilter": {"edges": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_none_persistent_graph(): @@ -1850,7 +959,8 @@ def test_graph_edge_property_filter_is_none_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1890,7 +1000,8 @@ def test_graph_edge_property_filter_is_some(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_some_persistent_graph(): @@ -1916,7 +1027,8 @@ def test_graph_edge_property_filter_is_some_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -1956,7 +1068,8 @@ def test_graph_edge_property_filter_is_in(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_in_persistent_graph(): @@ -1983,7 +1096,8 @@ def test_graph_edge_property_filter_is_in_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -2012,7 +1126,8 @@ def test_graph_edge_property_filter_is_empty_list(graph): } """ expected_output = {"graph": {"edgeFilter": {"edges": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_in_empty_list_persistent_graph(): @@ -2039,7 +1154,8 @@ def test_graph_edge_property_filter_is_in_empty_list_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -2066,7 +1182,8 @@ def test_graph_edge_property_filter_is_in_no_value_error(graph): } """ expected_error_message = "Expected a value for IsIn operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_is_in_type_error(): @@ -2093,7 +1210,8 @@ def test_graph_edge_property_filter_is_in_type_error(): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, Graph()) + graph = create_test_graph(Graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_is_in_type_error_persistent_graph(): @@ -2120,7 +1238,8 @@ def test_graph_edge_property_filter_is_in_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -2155,7 +1274,8 @@ def test_graph_edge_property_filter_is_not_in(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_not_in_persistent_graph(): @@ -2182,7 +1302,8 @@ def test_graph_edge_property_filter_is_not_in_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) # Edge property filter is not supported yet for PersistentGraph @@ -2223,7 +1344,8 @@ def test_graph_edge_property_filter_is_not_in_empty_list(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) def test_graph_edge_property_filter_is_not_in_empty_list_persistent_graph(): @@ -2250,7 +1372,8 @@ def test_graph_edge_property_filter_is_not_in_empty_list_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -2277,7 +1400,8 @@ def test_graph_edge_property_filter_is_not_in_no_value_error(graph): } """ expected_error_message = "Expected a value for IsNotIn operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_is_not_in_type_error(): @@ -2304,7 +1428,8 @@ def test_graph_edge_property_filter_is_not_in_type_error(): } """ expected_error_message = "PropertyType Error: Wrong type for property eprop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, Graph()) + graph = create_test_graph(Graph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_property_filter_is_not_in_type_error_persistent_graph(): @@ -2331,7 +1456,8 @@ def test_graph_edge_property_filter_is_not_in_type_error_persistent_graph(): } """ expected_error_message = "Property filtering not implemented on PersistentGraph yet" - run_graphql_error_test(query, expected_error_message, PersistentGraph()) + graph = create_test_graph(PersistentGraph()) + run_graphql_error_test(query, expected_error_message, graph) def test_graph_edge_not_property_filter(): @@ -2340,7 +1466,7 @@ def test_graph_edge_not_property_filter(): graph(path: "g") { edgeFilter ( filter: { - not: + not: { property: { name: "eprop5" @@ -2373,4 +1499,5 @@ def test_graph_edge_not_property_filter(): } } } - run_graphql_test(query, expected_output, Graph()) + graph = create_test_graph(Graph()) + run_graphql_test(query, expected_output, graph) diff --git a/python/tests/test_base_install/test_graphql/test_filters/test_graph_nodes_property_filter.py b/python/tests/test_base_install/test_graphql/test_filters/test_graph_nodes_property_filter.py new file mode 100644 index 0000000000..bfd243248e --- /dev/null +++ b/python/tests/test_base_install/test_graphql/test_filters/test_graph_nodes_property_filter.py @@ -0,0 +1,891 @@ +import pytest +from raphtory import Graph, PersistentGraph +from filters_setup import PORT, create_test_graph +from utils import run_graphql_test, run_graphql_error_test + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_equal(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop5" + operator: EQUAL + value: { list: [ {i64: 1}, {i64: 2}, {i64: 3} ] } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_equal_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop5" + operator: EQUAL + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for Equal operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_equal_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop5" + operator: EQUAL + value: { i64: 1 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop5: expected List(I64) but actual type is I64" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_not_equal(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop4" + operator: NOT_EQUAL + value: { bool: true } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_not_equal_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop4" + operator: NOT_EQUAL + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for NotEqual operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_not_equal_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop4" + operator: NOT_EQUAL + value: { i64: 1 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop4: expected Bool but actual type is I64" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than_or_equal(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN_OR_EQUAL + value: { i64: 60 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than_or_equal_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN_OR_EQUAL + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for GreaterThanOrEqual operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than_or_equal_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN_OR_EQUAL + value: { bool: true } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Bool" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than_or_equal(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN_OR_EQUAL + value: { i64: 30 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": { + "nodeFilter": { + "nodes": {"list": [{"name": "b"}, {"name": "c"}, {"name": "d"}]} + } + } + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than_or_equal_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN_OR_EQUAL + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for LessThanOrEqual operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than_or_equal_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN_OR_EQUAL + value: { str: "shivam" } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN + value: { i64: 30 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = {"graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}]}}}} + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for GreaterThan operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_greater_than_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: GREATER_THAN + value: { str: "shivam" } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN + value: { i64: 30 } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "c"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for LessThan operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_less_than_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: LESS_THAN + value: { str: "shivam" } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_none(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop5" + operator: IS_NONE + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_some(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop5" + operator: IS_SOME + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}, {"name": "c"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_in(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_IN + value: { list: [{i64: 10},{i64: 30},{i64: 50},{i64: 70}]} + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "b"}, {"name": "d"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_node_property_filter_is_in_empty_list(graph): + query = """ + query { + graph(path: "g") { + nodes { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_IN + value: { list: []} + } + } + ) { + list { + name + } + } + } + } + } + """ + expected_output = {"graph": {"nodes": {"nodeFilter": {"list": []}}}} + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_in_no_value(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_IN + value: { list: []} + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = {"graph": {"nodeFilter": {"nodes": {"list": []}}}} + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_in_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_IN + value: { str: "shivam" } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_not_in_any(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_NOT_IN + value: { list: [{i64: 10},{i64: 30},{i64: 50},{i64: 70}]} + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": {"nodeFilter": {"nodes": {"list": [{"name": "a"}, {"name": "c"}]}}} + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_node_property_filter_not_is_not_in_empty_list(graph): + query = """ + query { + graph(path: "g") { + nodes { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_NOT_IN + value: { list: []} + } + } + ) { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": { + "nodes": { + "nodeFilter": { + "list": [{"name": "a"}, {"name": "b"}, {"name": "c"}, {"name": "d"}] + } + } + } + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_not_in_no_value_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_NOT_IN + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "Expected a value for IsNotIn operator" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_property_filter_is_not_in_type_error(graph): + query = """ + query { + graph(path: "g") { + nodeFilter( + filter: { + property: { + name: "prop1" + operator: IS_NOT_IN + value: { str: "shivam" } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) + + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_not_property_filter(graph): + query = """ + query { + graph(path: "g") { + nodeFilter ( + filter: { + not: + { + property: { + name: "prop5" + operator: EQUAL + value: { list: [ {i64: 1}, {i64: 2} ] } + } + } + } + ) { + nodes { + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": { + "nodeFilter": { + "nodes": { + "list": [{"name": "a"}, {"name": "b"}, {"name": "c"}, {"name": "d"}] + } + } + } + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) + +@pytest.mark.parametrize("graph", [Graph, PersistentGraph]) +def test_graph_node_type_and_property_filter(graph): + query = """ + query { + graph(path: "g") { + nodes { + nodeFilter(filter: { + and: [{ + node: { + field: NODE_TYPE, + operator: IS_IN, + value: { + list: [ + {str: "fire_nation"}, + {str: "water_tribe"} + ] + } + } + },{ + property: { + name: "prop2", + operator: GREATER_THAN, + value: { f64:1 } + } + }] + }) { + count + list { + name + } + } + } + } + } + """ + expected_output = { + "graph": { + "nodes": { + "nodeFilter": { + "count": 3, + "list": [{"name": "a"}, {"name": "b"}, {"name": "c"}] + } + } + } + } + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) diff --git a/python/tests/test_base_install/test_graphql/test_nodes_property_filter.py b/python/tests/test_base_install/test_graphql/test_filters/test_nodes_property_filter.py similarity index 79% rename from python/tests/test_base_install/test_graphql/test_nodes_property_filter.py rename to python/tests/test_base_install/test_graphql/test_filters/test_nodes_property_filter.py index 3149b83087..a42e4f330f 100644 --- a/python/tests/test_base_install/test_graphql/test_nodes_property_filter.py +++ b/python/tests/test_base_install/test_graphql/test_filters/test_nodes_property_filter.py @@ -1,119 +1,7 @@ -import tempfile - import pytest - -from raphtory.graphql import GraphServer from raphtory import Graph, PersistentGraph -import json -import re - -PORT = 1737 - - -def create_test_graph(g): - g.add_node( - 1, - "a", - properties={ - "prop1": 60, - "prop2": 31.3, - "prop3": "abc123", - "prop4": True, - "prop5": [1, 2, 3], - }, - ) - g.add_node( - 1, - "b", - properties={"prop1": 10, "prop2": 31.3, "prop3": "abc223", "prop4": False}, - ) - g.add_node( - 1, - "c", - properties={ - "prop1": 20, - "prop2": 31.3, - "prop3": "abc333", - "prop4": True, - "prop5": [5, 6, 7], - }, - ) - g.add_node( - 1, - "d", - properties={"prop1": 30, "prop2": 31.3, "prop3": "abc444", "prop4": False}, - ) - g.add_edge( - 2, - "a", - "d", - properties={ - "eprop1": 60, - "eprop2": 0.4, - "eprop3": "xyz123", - "eprop4": True, - "eprop5": [1, 2, 3], - }, - ) - g.add_edge( - 2, - "b", - "d", - properties={ - "eprop1": 10, - "eprop2": 1.7, - "eprop3": "xyz123", - "eprop4": True, - "eprop5": [3, 4, 5], - }, - ) - g.add_edge( - 2, - "c", - "d", - properties={ - "eprop1": 30, - "eprop2": 6.4, - "eprop3": "xyz123", - "eprop4": False, - "eprop5": [10], - }, - ) - return g - - -def run_graphql_test(query, expected_output, graph): - create_test_graph(graph) - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(PORT) as server: - client = server.get_client() - client.send_graph(path="g", graph=graph) - - response = client.query(query) - - # Convert response to a dictionary if needed and compare - response_dict = json.loads(response) if isinstance(response, str) else response - assert response_dict == expected_output - - -def run_graphql_error_test(query, expected_error_message, graph): - create_test_graph(graph) - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(PORT) as server: - client = server.get_client() - client.send_graph(path="g", graph=graph) - - with pytest.raises(Exception) as excinfo: - client.query(query) - - full_error_message = str(excinfo.value) - match = re.search(r'"message":"(.*?)"', full_error_message) - error_message = match.group(1) if match else "" - - assert ( - error_message == expected_error_message - ), f"Expected '{expected_error_message}', but got '{error_message}'" - +from filters_setup import PORT, create_test_graph +from utils import run_graphql_test, run_graphql_error_test @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) def test_node_property_filter_equal(graph): @@ -139,7 +27,8 @@ def test_node_property_filter_equal(graph): } """ expected_output = {"graph": {"nodes": {"nodeFilter": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -165,7 +54,8 @@ def test_node_property_filter_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for Equal operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -192,7 +82,8 @@ def test_node_property_filter_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop5: expected List(I64) but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -221,7 +112,8 @@ def test_node_property_filter_not_equal(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "b"}, {"name": "d"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -247,7 +139,8 @@ def test_node_property_filter_not_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for NotEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -274,7 +167,8 @@ def test_node_property_filter_not_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop4: expected Bool but actual type is I64" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -301,7 +195,8 @@ def test_node_property_filter_greater_than_or_equal(graph): } """ expected_output = {"graph": {"nodes": {"nodeFilter": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -327,7 +222,8 @@ def test_node_property_filter_greater_than_or_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for GreaterThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -354,7 +250,8 @@ def test_node_property_filter_greater_than_or_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Bool" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -387,7 +284,8 @@ def test_node_property_filter_less_than_or_equal(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -413,7 +311,8 @@ def test_node_property_filter_less_than_or_equal_no_value_error(graph): } """ expected_error_message = "Expected a value for LessThanOrEqual operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -440,7 +339,8 @@ def test_node_property_filter_less_than_or_equal_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -467,7 +367,8 @@ def test_node_property_filter_greater_than(graph): } """ expected_output = {"graph": {"nodes": {"nodeFilter": {"list": [{"name": "a"}]}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -493,7 +394,8 @@ def test_node_property_filter_greater_than_no_value_error(graph): } """ expected_error_message = "Expected a value for GreaterThan operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -520,7 +422,8 @@ def test_node_property_filter_greater_than_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -549,7 +452,8 @@ def test_node_property_filter_less_than(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "b"}, {"name": "c"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -575,7 +479,8 @@ def test_node_property_filter_less_than_no_value_error(graph): } """ expected_error_message = "Expected a value for LessThan operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -602,7 +507,8 @@ def test_node_property_filter_less_than_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -630,7 +536,8 @@ def test_node_property_filter_is_none(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "b"}, {"name": "d"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -658,7 +565,8 @@ def test_node_property_filter_is_some(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "a"}, {"name": "c"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -687,7 +595,8 @@ def test_node_property_filter_is_in(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "b"}, {"name": "d"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -714,7 +623,8 @@ def test_node_property_filter_is_in_empty_list(graph): } """ expected_output = {"graph": {"nodes": {"nodeFilter": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -741,7 +651,8 @@ def test_node_property_filter_is_in_no_value(graph): } """ expected_output = {"graph": {"nodes": {"nodeFilter": {"list": []}}}} - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -768,7 +679,8 @@ def test_node_property_filter_is_in_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -797,7 +709,8 @@ def test_node_property_filter_is_not_in(graph): expected_output = { "graph": {"nodes": {"nodeFilter": {"list": [{"name": "a"}, {"name": "c"}]}}} } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -832,7 +745,8 @@ def test_node_property_filter_is_not_in_empty_list(graph): } } } - run_graphql_test(query, expected_output, graph()) + graph = create_test_graph(graph()) + run_graphql_test(query, expected_output, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -858,7 +772,8 @@ def test_node_property_filter_is_not_in_no_value_error(graph): } """ expected_error_message = "Expected a value for IsNotIn operator" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) @pytest.mark.parametrize("graph", [Graph, PersistentGraph]) @@ -885,4 +800,5 @@ def test_node_property_filter_is_not_in_type_error(graph): } """ expected_error_message = "PropertyType Error: Wrong type for property prop1: expected I64 but actual type is Str" - run_graphql_error_test(query, expected_error_message, graph()) + graph = create_test_graph(graph()) + run_graphql_error_test(query, expected_error_message, graph) diff --git a/python/tests/test_base_install/test_index_spec.py b/python/tests/test_base_install/test_index_spec.py new file mode 100644 index 0000000000..f4a2d169a2 --- /dev/null +++ b/python/tests/test_base_install/test_index_spec.py @@ -0,0 +1,180 @@ +from raphtory import Graph, IndexSpecBuilder +from raphtory import filter + +def init_graph(graph): + nodes = [ + (1, "pometry", {"p1": 5, "p2": 50}, "fire_nation", {"x": True}), + (1, "raphtory", {"p1": 10, "p2": 100}, "water_tribe", {"y": False}), + ] + for t, name, props, group, const_props in nodes: + n = graph.add_node(t, name, props, group) + n.add_constant_properties(const_props) + + edges = [ + (1, "pometry", "raphtory", {"e_p1": 3.2, "e_p2": 10.0}, "Football", {"e_x": True}), + (1, "raphtory", "pometry", {"e_p1": 4.0, "e_p2": 20.0}, "Baseball", {"e_y": False}), + ] + for t, src, dst, props, label, const_props in edges: + e = graph.add_edge(t, src, dst, props, label) + e.add_constant_properties(const_props, label) + + return graph + + +def search_nodes(graph, filter_expr): + return sorted(n.name for n in graph.search_nodes(filter_expr, 10, 0)) + + +def search_edges(graph, filter_expr): + return sorted(f"{e.src.name}->{e.dst.name}" for e in graph.search_edges(filter_expr, 10, 0)) + + +def test_with_all_props_index_spec(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_all_node_props() + .with_all_edge_props() + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("p1") == 5 + f2 = filter.Property("x") == True + assert search_nodes(graph, f1 & f2) == ["pometry"] + + f1 = filter.Property("e_p1") < 5.0 + f2 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1 & f2)) == sorted(["raphtory->pometry"]) + + +def test_with_selected_props_index_spec(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_const_node_props(["y"]) + .with_temp_node_props(["p1"]) + .with_const_edge_props(["e_y"]) + .with_temp_edge_props(["e_p1"]) + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("p1") == 5 + f2 = filter.Property("y") == False + assert sorted(search_nodes(graph, f1 | f2)) == sorted(["pometry", "raphtory"]) + + f = filter.Property("y") == False + assert search_nodes(graph, f) == ["raphtory"] + + f1 = filter.Property("e_p1") < 5.0 + f2 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"]) + + +def test_with_invalid_property_returns_error(): + graph = init_graph(Graph()) + try: + IndexSpecBuilder(graph).with_const_node_props(["xyz"]) + assert False, "Expected error for unknown property" + except Exception as e: + assert "xyz" in str(e) + + +def test_build_empty_spec_by_default(): + graph = init_graph(Graph()) + spec = IndexSpecBuilder(graph).build() + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("p1") == 5 + f2 = filter.Property("x") == True + assert sorted(search_nodes(graph, f1 & f2)) == ["pometry"] + + f1 = filter.Property("e_p1") < 5.0 + f2 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"]) + + +def test_mixed_node_and_edge_props_index_spec(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_const_node_props(["x"]) + .with_all_temp_node_props() + .with_all_edge_props() + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("p1") == 5 + f2 = filter.Property("y") == False + assert sorted(search_nodes(graph, f1 | f2)) == sorted(["pometry", "raphtory"]) + + f1 = filter.Property("e_p1") < 5.0 + f2 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"]) + + +def test_get_index_spec(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_const_node_props(["x"]) + .with_all_temp_node_props() + .with_all_edge_props() + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + returned_spec = graph.get_index_spec() + + node_const_names = {name for name in returned_spec.node_const_props} + node_temp_names = {name for name in returned_spec.node_temp_props} + edge_const_names = {name for name in returned_spec.edge_const_props} + edge_temp_names = {name for name in returned_spec.edge_temp_props} + + assert "x" in node_const_names + assert "p1" in node_temp_names or "p2" in node_temp_names + assert "e_x" in edge_const_names or "e_y" in edge_const_names + assert "e_p1" in edge_temp_names or "e_p2" in edge_temp_names + + +def test_const_prop_fallback_when_const_prop_indexed(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_const_node_props(["x"]) + .with_const_edge_props(["e_y"]) + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("x") == True + assert sorted(search_nodes(graph, f1)) == sorted(["pometry"]) + + f1 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1)) == sorted(["raphtory->pometry"]) + + +def test_const_prop_fallback_when_const_prop_not_indexed(): + graph = init_graph(Graph()) + spec = ( + IndexSpecBuilder(graph) + .with_all_temp_node_props() + .with_all_temp_edge_props() + .build() + ) + + graph.create_index_in_ram_with_spec(spec) + + f1 = filter.Property("x") == True + assert sorted(search_nodes(graph, f1)) == sorted(["pometry"]) + + f1 = filter.Property("e_y") == False + assert sorted(search_edges(graph, f1)) == sorted(["raphtory->pometry"]) diff --git a/raphtory-graphql/src/graph.rs b/raphtory-graphql/src/graph.rs index f526f25047..fea9e54ad2 100644 --- a/raphtory-graphql/src/graph.rs +++ b/raphtory-graphql/src/graph.rs @@ -21,7 +21,9 @@ use raphtory::{ }, graph::{edge::EdgeView, node::NodeView}, }, - prelude::{CacheOps, DeletionOps, EdgeViewOps, NodeViewOps, SearchableGraphOps}, + prelude::{ + CacheOps, DeletionOps, EdgeViewOps, IndexMutationOps, NodeViewOps, SearchableGraphOps, + }, serialise::GraphFolder, vectors::{cache::VectorCache, vectorised_graph::VectorisedGraph}, }; diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 24c3e38dda..76abee0440 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -132,14 +132,18 @@ mod graphql_test { node: { field: NODE_NAME, operator: EQUAL, - value: "N1" + value: { + str: "N1" + } } }, { node: { field: NODE_TYPE, operator: NOT_EQUAL, - value: "air_nomads" + value: { + str: "air_nomads" + } } }, { diff --git a/raphtory-graphql/src/model/graph/filtering.rs b/raphtory-graphql/src/model/graph/filtering.rs index 23a7a79f6b..0358cc8b64 100644 --- a/raphtory-graphql/src/model/graph/filtering.rs +++ b/raphtory-graphql/src/model/graph/filtering.rs @@ -148,6 +148,26 @@ pub enum Operator { NotContains, } +impl Display for Operator { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let op_str = match self { + Operator::Equal => "EQUAL", + Operator::NotEqual => "NOT_EQUAL", + Operator::GreaterThanOrEqual => "GREATER_THAN_OR_EQUAL", + Operator::LessThanOrEqual => "LESS_THAN_OR_EQUAL", + Operator::GreaterThan => "GREATER_THAN", + Operator::LessThan => "LESS_THAN", + Operator::IsNone => "IS_NONE", + Operator::IsSome => "IS_SOME", + Operator::IsIn => "IS_IN", + Operator::IsNotIn => "IS_NOT_IN", + Operator::Contains => "CONTAINS", + Operator::NotContains => "NOT_CONTAINS", + }; + write!(f, "{op_str}") + } +} + #[derive(InputObject, Clone, Debug)] pub struct NodeFilter { pub node: Option, @@ -219,7 +239,7 @@ impl NodeFilter { pub struct NodeFieldFilter { pub field: NodeField, pub operator: Operator, - pub value: String, + pub value: Value, } #[derive(Enum, Copy, Clone, Debug)] @@ -291,6 +311,33 @@ pub enum TemporalType { Latest, } +fn field_value(value: Value, operator: Operator) -> Result { + let prop = Prop::try_from(value.clone())?; + match (prop, operator) { + (Prop::List(list), Operator::IsIn | Operator::IsNotIn) => { + let strings: Vec = list + .iter() + .map(|p| match p { + Prop::Str(s) => Ok(s.to_string()), + _ => Err(GraphError::InvalidGqlFilter(format!( + "Invalid field value {:?} or operator {}", + value, operator + ))), + }) + .collect::>()?; + + Ok(FilterValue::Set(Arc::new( + strings.iter().cloned().collect(), + ))) + } + (Prop::Str(p), _) => Ok(FilterValue::Single(p.to_string())), + _ => Err(GraphError::InvalidGqlFilter(format!( + "Invalid field value {:?} or operator {}", + value, operator + ))), + } +} + impl TryFrom for CompositeNodeFilter { type Error = GraphError; @@ -300,7 +347,7 @@ impl TryFrom for CompositeNodeFilter { if let Some(node) = filter.node { exprs.push(CompositeNodeFilter::Node(Filter { field_name: node.field.to_string(), - field_value: FilterValue::Single(node.value), + field_value: field_value(node.value, node.operator)?, operator: node.operator.into(), })); } @@ -376,7 +423,7 @@ impl TryFrom for CompositeEdgeFilter { if let Some(src) = filter.src { exprs.push(CompositeEdgeFilter::Edge(Filter { field_name: "src".to_string(), - field_value: FilterValue::Single(src.value), + field_value: field_value(src.value, src.operator)?, operator: src.operator.into(), })); } @@ -384,7 +431,7 @@ impl TryFrom for CompositeEdgeFilter { if let Some(dst) = filter.dst { exprs.push(CompositeEdgeFilter::Edge(Filter { field_name: "dst".to_string(), - field_value: FilterValue::Single(dst.value), + field_value: field_value(dst.value, dst.operator)?, operator: dst.operator.into(), })); } diff --git a/raphtory/src/db/api/mutation/index_ops.rs b/raphtory/src/db/api/mutation/index_ops.rs new file mode 100644 index 0000000000..182b1ab669 --- /dev/null +++ b/raphtory/src/db/api/mutation/index_ops.rs @@ -0,0 +1,123 @@ +use crate::{ + core::utils::errors::GraphError, + db::api::view::{IndexSpec, IndexSpecBuilder}, + prelude::AdditionOps, +}; +use std::{ + fs::File, + path::{Path, PathBuf}, +}; +use zip::ZipArchive; + +pub trait IndexMutationOps: Sized + AdditionOps { + fn create_index(&self) -> Result<(), GraphError>; + + fn create_index_with_spec(&self, index_spec: IndexSpec) -> Result<(), GraphError>; + + fn create_index_in_ram(&self) -> Result<(), GraphError>; + + fn create_index_in_ram_with_spec(&self, index_spec: IndexSpec) -> Result<(), GraphError>; + + fn load_index(&self, path: &PathBuf) -> Result<(), GraphError>; + + fn persist_index_to_disk(&self, path: &PathBuf) -> Result<(), GraphError>; + + fn persist_index_to_disk_zip(&self, path: &PathBuf) -> Result<(), GraphError>; +} + +impl IndexMutationOps for G { + fn create_index(&self) -> Result<(), GraphError> { + let index_spec = IndexSpecBuilder::new(self.clone()) + .with_all_node_props() + .with_all_edge_props() + .build(); + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.get_or_create_index(index_spec)?; + Ok(()) + }) + } + + fn create_index_with_spec(&self, index_spec: IndexSpec) -> Result<(), GraphError> { + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.get_or_create_index(index_spec)?; + Ok(()) + }) + } + + fn create_index_in_ram(&self) -> Result<(), GraphError> { + let index_spec = IndexSpecBuilder::new(self.clone()) + .with_all_node_props() + .with_all_edge_props() + .build(); + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.get_or_create_index_in_ram(index_spec)?; + Ok(()) + }) + } + + fn create_index_in_ram_with_spec(&self, index_spec: IndexSpec) -> Result<(), GraphError> { + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.get_or_create_index_in_ram(index_spec)?; + Ok(()) + }) + } + + fn load_index(&self, path: &PathBuf) -> Result<(), GraphError> { + fn has_index>(zip_path: P) -> Result { + let file = File::open(&zip_path)?; + let mut archive = ZipArchive::new(file)?; + + for i in 0..archive.len() { + let entry = archive.by_index(i)?; + let entry_path = Path::new(entry.name()); + + if let Some(first_component) = entry_path.components().next() { + if first_component.as_os_str() == "index" { + return Ok(true); + } + } + } + + Ok(false) + } + + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + if path.is_file() { + if has_index(path)? { + storage.get_or_load_index(path.clone())?; + } else { + return Ok(()); // Skip if no index in zip + } + } else { + let index_path = path.join("index"); + if index_path.exists() && index_path.read_dir()?.next().is_some() { + storage.get_or_load_index(path.clone())?; + } + } + + Ok(()) + }) + } + + fn persist_index_to_disk(&self, path: &PathBuf) -> Result<(), GraphError> { + let path = path.join("index"); + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.persist_index_to_disk(&path)?; + Ok(()) + }) + } + + fn persist_index_to_disk_zip(&self, path: &PathBuf) -> Result<(), GraphError> { + self.get_storage() + .map_or(Err(GraphError::IndexingNotSupported), |storage| { + storage.persist_index_to_disk_zip(&path)?; + Ok(()) + }) + } +} diff --git a/raphtory/src/db/api/mutation/mod.rs b/raphtory/src/db/api/mutation/mod.rs index e813cbd8e6..00e61ada42 100644 --- a/raphtory/src/db/api/mutation/mod.rs +++ b/raphtory/src/db/api/mutation/mod.rs @@ -1,3 +1,4 @@ +use self::internal::InternalAdditionOps; use crate::{ core::{ utils::{ @@ -8,20 +9,22 @@ use crate::{ }, prelude::Prop, }; +use raphtory_api::core::storage::timeindex::{AsTime, TimeIndexEntry}; mod addition_ops; mod deletion_ops; mod import_ops; +#[cfg(feature = "search")] +pub mod index_ops; pub mod internal; mod property_addition_ops; pub use addition_ops::AdditionOps; pub use deletion_ops::DeletionOps; pub use import_ops::ImportOps; +#[cfg(feature = "search")] +pub use index_ops::IndexMutationOps; pub use property_addition_ops::PropertyAdditionOps; -use raphtory_api::core::storage::timeindex::{AsTime, TimeIndexEntry}; - -use self::internal::InternalAdditionOps; /// Used to handle automatic injection of secondary index if not explicitly provided pub enum InputTime { diff --git a/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs index 644d78ed8b..fbd2f064ff 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs @@ -1251,7 +1251,7 @@ mod test_graph_storage { use super::*; use crate::{ db::{api::view::SearchableGraphOps, graph::views::filter::model::PropertyFilterOps}, - prelude::{Graph, NodeViewOps, PropertyFilter}, + prelude::{Graph, IndexMutationOps, NodeViewOps, PropertyFilter}, }; #[test] @@ -1277,8 +1277,9 @@ mod test_graph_storage { use super::*; use crate::{ db::{api::view::SearchableGraphOps, graph::views::filter::model::PropertyFilterOps}, - prelude::{EdgeViewOps, Graph, NodeViewOps, PropertyFilter}, + prelude::{EdgeViewOps, Graph, IndexMutationOps, NodeViewOps, PropertyFilter}, }; + #[test] fn test_search_edges_latest() { let g = Graph::new(); diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index ccd508f9f7..83b92aad34 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -21,9 +21,9 @@ use crate::{ }, }; -use crate::db::api::{ - storage::graph::edges::edge_storage_ops::EdgeStorageOps, - view::internal::{InheritEdgeHistoryFilter, InheritNodeHistoryFilter, InternalStorageOps}, +use crate::db::api::view::{ + internal::{InheritEdgeHistoryFilter, InheritNodeHistoryFilter, InternalStorageOps}, + IndexSpec, }; #[cfg(feature = "search")] use crate::search::graph_index::GraphIndex; @@ -140,33 +140,45 @@ impl Storage { #[cfg(feature = "search")] impl Storage { + pub(crate) fn get_index_spec(&self) -> Result { + let index = self.index.get().ok_or(GraphError::GraphIndexIsMissing)?; + Ok(index.index_spec.read().clone()) + } + + pub(crate) fn get_or_load_index(&self, path: PathBuf) -> Result<&GraphIndex, GraphError> { + self.index.get_or_try_init(|| { + let index = GraphIndex::load_from_path(&path)?; + Ok(index) + }) + } + pub(crate) fn get_or_create_index( &self, - path: Option, + index_spec: IndexSpec, ) -> Result<&GraphIndex, GraphError> { - self.index.get_or_try_init(|| { - if let Some(path) = path { - Ok::<_, GraphError>(GraphIndex::load_from_path(&path)?) - } else { - let cache_path = self.get_cache().map(|cache| cache.folder.get_base_path()); - Ok::<_, GraphError>(GraphIndex::create_from_graph( - &self.graph, - false, - cache_path, - )?) - } - }) + let index = self.index.get_or_try_init(|| { + let cached_graph_path = self.get_cache().map(|cache| cache.folder.get_base_path()); + GraphIndex::create(&self.graph, false, cached_graph_path, IndexSpec::default()) + })?; + index.update(&self.graph, index_spec.clone())?; + Ok(index) } - pub(crate) fn get_or_create_index_in_ram(&self) -> Result<&GraphIndex, GraphError> { + pub(crate) fn get_or_create_index_in_ram( + &self, + index_spec: IndexSpec, + ) -> Result<&GraphIndex, GraphError> { let index = self.index.get_or_try_init(|| { - Ok::<_, GraphError>(GraphIndex::create_from_graph(&self.graph, true, None)?) + GraphIndex::create(&self.graph, true, None, IndexSpec::default()) })?; + if index.path.is_some() { - Err(GraphError::FailedToCreateIndexInRam) - } else { - Ok(index) + return Err(GraphError::FailedToCreateIndexInRam); } + + index.update(&self.graph, index_spec)?; + + Ok(index) } pub(crate) fn get_index(&self) -> Option<&GraphIndex> { @@ -316,9 +328,6 @@ impl InternalAdditionOps for Storage { #[cfg(feature = "proto")] self.if_cache(|cache| cache.resolve_node_property(prop, id, &dtype, is_static)); - #[cfg(feature = "search")] - self.if_index(|index| index.create_node_property_index(id, prop, &dtype, is_static))?; - Ok(id) } @@ -335,9 +344,6 @@ impl InternalAdditionOps for Storage { #[cfg(feature = "proto")] self.if_cache(|cache| cache.resolve_edge_property(prop, id, &dtype, is_static)); - #[cfg(feature = "search")] - self.if_index(|index| index.create_edge_property_index(id, prop, &dtype, is_static))?; - Ok(id) } @@ -375,7 +381,7 @@ impl InternalAdditionOps for Storage { }); #[cfg(feature = "search")] - self.if_index(|index| index.add_edge_update(&self.graph, id, t, src, dst, layer, props))?; + self.if_index(|index| index.add_edge_update(&self.graph, id, t, layer, props))?; Ok(id) } @@ -393,12 +399,7 @@ impl InternalAdditionOps for Storage { self.if_cache(|cache| cache.add_edge_update(t, edge, props, layer)); #[cfg(feature = "search")] - self.if_index(|index| { - let ee = self.graph.edge_entry(edge); - let src = ee.src(); - let dst = ee.dst(); - index.add_edge_update(&self.graph, Existing(edge), t, src, dst, layer, props) - })?; + self.if_index(|index| index.add_edge_update(&self.graph, Existing(edge), t, layer, props))?; Ok(()) } @@ -451,7 +452,7 @@ impl InternalPropertyAdditionOps for Storage { self.if_cache(|cache| cache.add_node_cprops(vid, props)); #[cfg(feature = "search")] - self.if_index(|index| index.add_node_constant_properties(&self.graph, vid, props))?; + self.if_index(|index| index.add_node_constant_properties(vid, props))?; Ok(()) } @@ -468,7 +469,7 @@ impl InternalPropertyAdditionOps for Storage { self.if_cache(|cache| cache.add_node_cprops(vid, props)); #[cfg(feature = "search")] - self.if_index(|index| index.update_node_constant_properties(&self.graph, vid, props))?; + self.if_index(|index| index.update_node_constant_properties(vid, props))?; Ok(()) } @@ -486,7 +487,7 @@ impl InternalPropertyAdditionOps for Storage { self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); #[cfg(feature = "search")] - self.if_index(|index| index.add_edge_constant_properties(&self.graph, eid, layer, props))?; + self.if_index(|index| index.add_edge_constant_properties(eid, layer, props))?; Ok(()) } @@ -504,9 +505,7 @@ impl InternalPropertyAdditionOps for Storage { self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); #[cfg(feature = "search")] - self.if_index(|index| { - index.update_edge_constant_properties(&self.graph, eid, layer, props) - })?; + self.if_index(|index| index.update_edge_constant_properties(eid, layer, props))?; Ok(()) } diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 339ee23e69..d163b28496 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -37,7 +37,7 @@ use chrono::{DateTime, Utc}; use raphtory_api::{ atomic_extra::atomic_usize_from_mut_slice, core::{ - entities::EID, + entities::{properties::props::PropMapper, EID}, storage::{arc_str::ArcStr, timeindex::TimeIndexEntry}, Direction, }, @@ -45,12 +45,9 @@ use raphtory_api::{ use rayon::prelude::*; use rustc_hash::FxHashSet; use std::{ - fs::File, - path::{Path, PathBuf}, + collections::HashSet, sync::{atomic::Ordering, Arc}, }; -#[cfg(feature = "search")] -use zip::ZipArchive; /// This trait GraphViewOps defines operations for accessing /// information about a graph. The trait has associated types @@ -138,15 +135,7 @@ pub trait GraphViewOps<'graph>: BoxableGraphView + Sized + Clone + 'graph { #[cfg(feature = "search")] pub trait SearchableGraphOps: Sized { - fn create_index(&self) -> Result<(), GraphError>; - - fn create_index_in_ram(&self) -> Result<(), GraphError>; - - fn load_index(&self, path: &PathBuf) -> Result<(), GraphError>; - - fn persist_index_to_disk(&self, path: &PathBuf) -> Result<(), GraphError>; - - fn persist_index_to_disk_zip(&self, path: &PathBuf) -> Result<(), GraphError>; + fn get_index_spec(&self) -> Result; fn search_nodes( &self, @@ -627,76 +616,221 @@ impl<'graph, G: BoxableGraphView + Sized + Clone + 'graph> GraphViewOps<'graph> } } -#[cfg(feature = "search")] -impl SearchableGraphOps for G { - fn create_index(&self) -> Result<(), GraphError> { - self.get_storage() - .map_or(Err(GraphError::IndexingNotSupported), |storage| { - storage.get_or_create_index(None)?; - Ok(()) - }) - } +#[derive(Debug, Clone, PartialEq, Default)] +pub struct IndexSpec { + pub(crate) node_const_props: HashSet, + pub(crate) node_temp_props: HashSet, + pub(crate) edge_const_props: HashSet, + pub(crate) edge_temp_props: HashSet, +} - fn create_index_in_ram(&self) -> Result<(), GraphError> { - self.get_storage() - .map_or(Err(GraphError::IndexingNotSupported), |storage| { - storage.get_or_create_index_in_ram()?; - Ok(()) +impl IndexSpec { + pub(crate) fn diff(existing: &IndexSpec, requested: &IndexSpec) -> Option { + fn diff_props(existing: &HashSet, requested: &HashSet) -> HashSet { + requested.difference(existing).copied().collect() + } + + let node_const_props = diff_props(&existing.node_const_props, &requested.node_const_props); + let node_temp_props = diff_props(&existing.node_temp_props, &requested.node_temp_props); + let edge_const_props = diff_props(&existing.edge_const_props, &requested.edge_const_props); + let edge_temp_props = diff_props(&existing.edge_temp_props, &requested.edge_temp_props); + + if node_const_props.is_empty() + && node_temp_props.is_empty() + && edge_const_props.is_empty() + && edge_temp_props.is_empty() + { + None + } else { + Some(IndexSpec { + node_const_props, + node_temp_props, + edge_const_props, + edge_temp_props, }) + } } - fn load_index(&self, path: &PathBuf) -> Result<(), GraphError> { - fn has_index>(zip_path: P) -> Result { - let file = File::open(&zip_path)?; - let mut archive = ZipArchive::new(file)?; + pub(crate) fn union(existing: &IndexSpec, other: &IndexSpec) -> IndexSpec { + fn union_props(a: &HashSet, b: &HashSet) -> HashSet { + a.union(b).copied().collect() + } - for i in 0..archive.len() { - let entry = archive.by_index(i)?; - let entry_path = Path::new(entry.name()); + IndexSpec { + node_const_props: union_props(&existing.node_const_props, &other.node_const_props), + node_temp_props: union_props(&existing.node_temp_props, &other.node_temp_props), + edge_const_props: union_props(&existing.edge_const_props, &other.edge_const_props), + edge_temp_props: union_props(&existing.edge_temp_props, &other.edge_temp_props), + } + } +} - if let Some(first_component) = entry_path.components().next() { - if first_component.as_os_str() == "index" { - return Ok(true); - } - } - } +#[derive(Clone)] +pub struct IndexSpecBuilder { + pub graph: G, + node_const_props: Option>, + node_temp_props: Option>, + edge_const_props: Option>, + edge_temp_props: Option>, +} - Ok(false) +impl IndexSpecBuilder { + pub fn new(graph: G) -> Self { + Self { + graph, + node_const_props: None, + node_temp_props: None, + edge_const_props: None, + edge_temp_props: None, } + } - self.get_storage() - .map_or(Err(GraphError::IndexingNotSupported), |storage| { - if path.is_file() { - if has_index(path)? { - storage.get_or_create_index(Some(path.clone()))?; - } else { - return Ok(()); // Skip if no index in zip - } - } else { - let index_path = path.join("index"); - if index_path.exists() && index_path.read_dir()?.next().is_some() { - storage.get_or_create_index(Some(path.clone()))?; - } - } + pub fn with_all_node_props(mut self) -> Self { + self.node_const_props = Some(Self::extract_props( + self.graph.node_meta().const_prop_meta(), + )); + self.node_temp_props = Some(Self::extract_props( + self.graph.node_meta().temporal_prop_meta(), + )); + self + } - Ok(()) + pub fn with_all_const_node_props(mut self) -> Self { + self.node_const_props = Some(Self::extract_props( + self.graph.node_meta().const_prop_meta(), + )); + self + } + + pub fn with_all_temp_node_props(mut self) -> Self { + self.node_temp_props = Some(Self::extract_props( + self.graph.node_meta().temporal_prop_meta(), + )); + self + } + + pub fn with_const_node_props>( + mut self, + props: impl IntoIterator, + ) -> Result { + self.node_const_props = Some(Self::extract_named_props( + self.graph.node_meta().const_prop_meta(), + props, + )?); + Ok(self) + } + + pub fn with_temp_node_props>( + mut self, + props: impl IntoIterator, + ) -> Result { + self.node_temp_props = Some(Self::extract_named_props( + self.graph.node_meta().temporal_prop_meta(), + props, + )?); + Ok(self) + } + + pub fn with_all_edge_props(mut self) -> Self { + self.edge_const_props = Some(Self::extract_props( + self.graph.edge_meta().const_prop_meta(), + )); + self.edge_temp_props = Some(Self::extract_props( + self.graph.edge_meta().temporal_prop_meta(), + )); + self + } + + pub fn with_all_edge_const_props(mut self) -> Self { + self.edge_const_props = Some(Self::extract_props( + self.graph.edge_meta().const_prop_meta(), + )); + self + } + + pub fn with_all_temp_edge_props(mut self) -> Self { + self.edge_temp_props = Some(Self::extract_props( + self.graph.edge_meta().temporal_prop_meta(), + )); + self + } + + pub fn with_const_edge_props>( + mut self, + props: impl IntoIterator, + ) -> Result { + self.edge_const_props = Some(Self::extract_named_props( + self.graph.edge_meta().const_prop_meta(), + props, + )?); + Ok(self) + } + + pub fn with_temp_edge_props>( + mut self, + props: impl IntoIterator, + ) -> Result { + self.edge_temp_props = Some(Self::extract_named_props( + self.graph.edge_meta().temporal_prop_meta(), + props, + )?); + Ok(self) + } + + fn extract_props(meta: &PropMapper) -> HashSet { + meta.get_keys() + .into_iter() + .filter_map(|k| { + meta.get_id(&*k) + .and_then(|id| meta.get_dtype(id).map(|_| id)) }) + .collect() } - fn persist_index_to_disk(&self, path: &PathBuf) -> Result<(), GraphError> { - let path = path.join("index"); - self.get_storage() - .map_or(Err(GraphError::IndexingNotSupported), |storage| { - storage.persist_index_to_disk(&path)?; - Ok(()) + fn extract_named_props>( + meta: &PropMapper, + keys: impl IntoIterator, + ) -> Result, GraphError> { + keys.into_iter() + .map(|k| { + let k: ArcStr = k.into(); + let key = k.to_string(); + let id = meta + .get_id(&*k) + .ok_or_else(|| GraphError::PropertyMissingError(key.clone()))?; + Ok(id) }) + .collect() } - fn persist_index_to_disk_zip(&self, path: &PathBuf) -> Result<(), GraphError> { + pub fn build(&self) -> IndexSpec { + IndexSpec { + node_const_props: match &self.node_const_props { + Some(props) => props.clone(), + None => HashSet::new(), + }, + node_temp_props: match &self.node_temp_props { + Some(props) => props.clone(), + None => HashSet::new(), + }, + edge_const_props: match &self.edge_const_props { + Some(props) => props.clone(), + None => HashSet::new(), + }, + edge_temp_props: match &self.edge_temp_props { + Some(props) => props.clone(), + None => HashSet::new(), + }, + } + } +} + +#[cfg(feature = "search")] +impl SearchableGraphOps for G { + fn get_index_spec(&self) -> Result { self.get_storage() .map_or(Err(GraphError::IndexingNotSupported), |storage| { - storage.persist_index_to_disk_zip(&path)?; - Ok(()) + storage.get_index_spec() }) } diff --git a/raphtory/src/db/graph/assertions.rs b/raphtory/src/db/graph/assertions.rs index 40655de336..e0fd3a7968 100644 --- a/raphtory/src/db/graph/assertions.rs +++ b/raphtory/src/db/graph/assertions.rs @@ -1,5 +1,7 @@ #[cfg(feature = "search")] pub use crate::db::api::view::SearchableGraphOps; +#[cfg(feature = "search")] +use crate::prelude::IndexMutationOps; use crate::{ db::{ api::view::StaticGraphViewOps, @@ -76,8 +78,6 @@ impl ApplyFilter for SearchNode fn apply(&self, graph: G) -> Vec { #[cfg(feature = "search")] { - graph.create_index_in_ram().unwrap(); - let mut results = graph .search_nodes(self.0.clone(), 20, 0) .unwrap() @@ -114,8 +114,6 @@ impl ApplyFilter for SearchEdge fn apply(&self, graph: G) -> Vec { #[cfg(feature = "search")] { - graph.create_index_in_ram().unwrap(); - let mut results = graph .search_edges(self.0.clone(), 20, 0) .unwrap() @@ -139,6 +137,7 @@ pub fn assert_filter_nodes_results( ) { assert_results( init_graph, + |_graph: &Graph| (), transform, expected, variants.into(), @@ -157,6 +156,7 @@ pub fn assert_search_nodes_results( { assert_results( init_graph, + |graph: &Graph| graph.create_index_in_ram().unwrap(), transform, expected, variants.into(), @@ -174,6 +174,7 @@ pub fn assert_filter_edges_results( ) { assert_results( init_graph, + |_graph: &Graph| (), transform, expected, variants.into(), @@ -192,6 +193,7 @@ pub fn assert_search_edges_results( { assert_results( init_graph, + |graph: &Graph| graph.create_index_in_ram().unwrap(), transform, expected, variants.into(), @@ -202,6 +204,7 @@ pub fn assert_search_edges_results( fn assert_results( init_graph: impl FnOnce(Graph) -> Graph, + pre_transform: impl Fn(&Graph) -> (), transform: impl GraphTransformer, expected: &[&str], variants: Vec, @@ -211,11 +214,13 @@ fn assert_results( for v in variants { match v { TestGraphVariants::Graph => { + pre_transform(&graph); let graph = transform.apply(graph.clone()); let result = apply.apply(graph); assert_eq!(expected, result); } TestGraphVariants::PersistentGraph => { + pre_transform(&graph); let base = graph.persistent_graph(); let graph = transform.apply(base); let result = apply.apply(graph); @@ -229,6 +234,7 @@ fn assert_results( let graph = DiskGraphStorage::from_graph(&graph, &tmp) .unwrap() .into_graph(); + pre_transform(&graph); let graph = transform.apply(graph); let result = apply.apply(graph); assert_eq!(expected, result); @@ -239,9 +245,10 @@ fn assert_results( { use crate::disk_graph::DiskGraphStorage; let tmp = TempDir::new().unwrap(); - let graph = DiskGraphStorage::from_graph(&graph, &tmp) - .unwrap() - .into_persistent_graph(); + let graph = DiskGraphStorage::from_graph(&graph, &tmp).unwrap(); + let graph = graph.into_graph(); + pre_transform(&graph); + let graph = graph.persistent_graph(); let graph = transform.apply(graph); let result = apply.apply(graph); assert_eq!(expected, result); @@ -250,3 +257,27 @@ fn assert_results( } } } + +#[cfg(feature = "search")] +pub fn search_nodes(graph: &Graph, filter: impl AsNodeFilter) -> Vec { + let mut results = graph + .search_nodes(filter, 10, 0) + .expect("Failed to search for nodes") + .into_iter() + .map(|v| v.name()) + .collect::>(); + results.sort(); + results +} + +#[cfg(feature = "search")] +pub fn search_edges(graph: &Graph, filter: impl AsEdgeFilter) -> Vec { + let mut results = graph + .search_edges(filter, 10, 0) + .expect("Failed to search for nodes") + .into_iter() + .map(|e| format!("{}->{}", e.src().name(), e.dst().name())) + .collect::>(); + results.sort(); + results +} diff --git a/raphtory/src/db/graph/views/filter/mod.rs b/raphtory/src/db/graph/views/filter/mod.rs index 3b65b7def3..42a4ea8215 100644 --- a/raphtory/src/db/graph/views/filter/mod.rs +++ b/raphtory/src/db/graph/views/filter/mod.rs @@ -893,14 +893,20 @@ pub(crate) mod test_filters { prelude::{AdditionOps, PropertyAdditionOps}, }; - use crate::db::graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, - TestVariants, - }, - views::filter::{ - model::property_filter::PropertyFilter, test_filters::IdentityGraphTransformer, + use crate::db::graph::views::filter::test_filters::InternalEdgeFilterOps; + + use crate::{ + db::graph::{ + assertions::{ + assert_filter_edges_results, assert_search_edges_results, + TestGraphVariants, TestVariants, + }, + views::filter::{ + model::property_filter::PropertyFilter, + test_filters::IdentityGraphTransformer, + }, }, + prelude::{EdgePropertyFilterOps, EdgeViewOps, Graph, GraphViewOps, NodeViewOps}, }; fn init_graph< @@ -1016,6 +1022,59 @@ pub(crate) mod test_filters { ); } + #[test] + #[ignore] + // TODO: Enable test once issue is fixed: https://github.com/Pometry/Raphtory/issues/2109 + fn test_constant_semantics2() { + fn filter_edges(graph: &Graph, filter: impl InternalEdgeFilterOps) -> Vec { + let mut results = graph + .filter_edges(filter) + .unwrap() + .edges() + .iter() + .map(|e| format!("{}->{}", e.src().name(), e.dst().name())) + .collect::>(); + results.sort(); + results + } + + let graph = init_graph(Graph::new()); + + let filter = PropertyFilter::property("p1").constant().eq(1u64); + assert_eq!( + filter_edges(&graph, filter.clone()), + vec![ + "N1->N2", "N10->N11", "N11->N12", "N12->N13", "N13->N14", "N14->N15", + "N15->N1", "N9->N10" + ] + ); + + let edge = graph + .add_edge(1, "shivam", "kapoor", [("p1", 100u64)], Some("fire_nation")) + .unwrap(); + edge.add_constant_properties([("z", true)], Some("fire_nation")) + .unwrap(); + let prop = graph + .edge("shivam", "kapoor") + .unwrap() + .properties() + .constant() + .get("z") + .unwrap(); + assert_eq!("{\"fire_nation\": true}", prop.to_string()); + + let filter2 = PropertyFilter::property("z").constant().eq(true); + assert_eq!(filter_edges(&graph, filter2), vec!["shivam->kapoor"]); + + assert_eq!( + filter_edges(&graph, filter), + vec![ + "N1->N2", "N10->N11", "N11->N12", "N12->N13", "N13->N14", "N14->N15", + "N15->N1", "N9->N10" + ] + ); + } + #[test] fn test_temporal_any_semantics() { // TODO: PropertyFilteringNotImplemented for variants persistent_graph, persistent_disk_graph for filter_edges. diff --git a/raphtory/src/lib.rs b/raphtory/src/lib.rs index eb166d2f2d..253af41ca2 100644 --- a/raphtory/src/lib.rs +++ b/raphtory/src/lib.rs @@ -144,7 +144,7 @@ pub mod prelude { }; #[cfg(feature = "search")] - pub use crate::db::api::view::SearchableGraphOps; + pub use crate::db::api::{mutation::IndexMutationOps, view::SearchableGraphOps}; } #[cfg(feature = "storage")] diff --git a/raphtory/src/python/graph/graph.rs b/raphtory/src/python/graph/graph.rs index d53a5a1305..1e9bd9a8f0 100644 --- a/raphtory/src/python/graph/graph.rs +++ b/raphtory/src/python/graph/graph.rs @@ -14,8 +14,8 @@ use crate::{ prelude::*, python::{ graph::{ - edge::PyEdge, graph_with_deletions::PyPersistentGraph, io::pandas_loaders::*, - node::PyNode, views::graph_view::PyGraphView, + edge::PyEdge, graph_with_deletions::PyPersistentGraph, index::PyIndexSpec, + io::pandas_loaders::*, node::PyNode, views::graph_view::PyGraphView, }, types::iterable::FromIterable, utils::{PyNodeRef, PyTime}, @@ -989,4 +989,31 @@ impl PyGraph { layer_col, ) } + + /// Create graph index + fn create_index(&self) -> Result<(), GraphError> { + self.graph.create_index() + } + + /// Create graph index with the provided index spec. + fn create_index_with_spec(&self, py_spec: &PyIndexSpec) -> Result<(), GraphError> { + self.graph.create_index_with_spec(py_spec.spec.clone()) + } + + /// Creates a graph index in memory (RAM). + /// + /// This is primarily intended for use in tests and should not be used in production environments, + /// as the index will not be persisted to disk. + fn create_index_in_ram(&self) -> Result<(), GraphError> { + self.graph.create_index_in_ram() + } + + /// Creates a graph index in memory (RAM) with the provided index spec. + /// + /// This is primarily intended for use in tests and should not be used in production environments, + /// as the index will not be persisted to disk. + fn create_index_in_ram_with_spec(&self, py_spec: &PyIndexSpec) -> Result<(), GraphError> { + self.graph + .create_index_in_ram_with_spec(py_spec.spec.clone()) + } } diff --git a/raphtory/src/python/graph/graph_with_deletions.rs b/raphtory/src/python/graph/graph_with_deletions.rs index 205b00a8c1..09c82eb02f 100644 --- a/raphtory/src/python/graph/graph_with_deletions.rs +++ b/raphtory/src/python/graph/graph_with_deletions.rs @@ -21,9 +21,9 @@ use crate::{ graph::{edge::EdgeView, node::NodeView, views::deletion_graph::PersistentGraph}, }, io::parquet_loaders::*, - prelude::{DeletionOps, GraphViewOps, ImportOps}, + prelude::{DeletionOps, GraphViewOps, ImportOps, IndexMutationOps}, python::{ - graph::{edge::PyEdge, node::PyNode, views::graph_view::PyGraphView}, + graph::{edge::PyEdge, index::PyIndexSpec, node::PyNode, views::graph_view::PyGraphView}, utils::{PyNodeRef, PyTime}, }, serialise::StableEncode, @@ -982,4 +982,31 @@ impl PyPersistentGraph { layer_col, ) } + + /// Create graph index + fn create_index(&self) -> Result<(), GraphError> { + self.graph.create_index() + } + + /// Create graph index with the provided index spec. + fn create_index_with_spec(&self, py_spec: &PyIndexSpec) -> Result<(), GraphError> { + self.graph.create_index_with_spec(py_spec.spec.clone()) + } + + /// Creates a graph index in memory (RAM). + /// + /// This is primarily intended for use in tests and should not be used in production environments, + /// as the index will not be persisted to disk. + fn create_index_in_ram(&self) -> Result<(), GraphError> { + self.graph.create_index_in_ram() + } + + /// Creates a graph index in memory (RAM) with the provided index spec. + /// + /// This is primarily intended for use in tests and should not be used in production environments, + /// as the index will not be persisted to disk. + fn create_index_in_ram_with_spec(&self, py_spec: &PyIndexSpec) -> Result<(), GraphError> { + self.graph + .create_index_in_ram_with_spec(py_spec.spec.clone()) + } } diff --git a/raphtory/src/python/graph/index.rs b/raphtory/src/python/graph/index.rs index 68b957816d..9da44c5fcf 100644 --- a/raphtory/src/python/graph/index.rs +++ b/raphtory/src/python/graph/index.rs @@ -1,27 +1,187 @@ use crate::{ core::utils::errors::GraphError, db::{ - api::view::internal::DynamicGraph, + api::view::{ + internal::{CoreGraphOps, DynamicGraph}, + IndexSpec, IndexSpecBuilder, IntoDynamic, MaterializedGraph, + }, graph::{edge::EdgeView, node::NodeView}, }, prelude::SearchableGraphOps, python::{graph::views::graph_view::PyGraphView, types::wrappers::filter_expr::PyFilterExpr}, }; use pyo3::prelude::*; +use raphtory_api::core::entities::properties::props::PropMapper; +use std::collections::HashSet; + +#[pyclass(name = "IndexSpec", module = "raphtory", frozen)] +#[derive(Clone)] +pub struct PyIndexSpec { + pub(crate) graph: DynamicGraph, + pub(crate) spec: IndexSpec, +} #[pymethods] -impl PyGraphView { - /// Create graph index - fn create_index(&self) -> Result<(), GraphError> { - self.graph.create_index() +impl PyIndexSpec { + fn __repr__(&self) -> PyResult { + let repr = format!( + "IndexSpec(\n node_const_props=[{}],\n node_temp_props=[{}],\n edge_const_props=[{}],\n edge_temp_props=[{}]\n)", + self.prop_repr(&self.spec.node_const_props, self.node_const_meta()), + self.prop_repr(&self.spec.node_temp_props, self.node_temp_meta()), + self.prop_repr(&self.spec.edge_const_props, self.edge_const_meta()), + self.prop_repr(&self.spec.edge_temp_props, self.edge_temp_meta()), + ); + Ok(repr) } - /// Creates a graph index in memory (RAM). - /// - /// This is primarily intended for use in tests and should not be used in production environments, - /// as the index will not be persisted to disk. - fn create_index_in_ram(&self) -> Result<(), GraphError> { - self.graph.create_index_in_ram() + #[getter] + fn node_const_props(&self) -> Vec { + self.prop_names(&self.spec.node_const_props, self.node_const_meta()) + } + + #[getter] + fn node_temp_props(&self) -> Vec { + self.prop_names(&self.spec.node_temp_props, self.node_temp_meta()) + } + + #[getter] + fn edge_const_props(&self) -> Vec { + self.prop_names(&self.spec.edge_const_props, self.edge_const_meta()) + } + + #[getter] + fn edge_temp_props(&self) -> Vec { + self.prop_names(&self.spec.edge_temp_props, self.edge_temp_meta()) + } +} + +impl PyIndexSpec { + fn prop_names(&self, prop_ids: &HashSet, meta: &PropMapper) -> Vec { + let mut names: Vec = prop_ids + .iter() + .map(|id| meta.get_name(*id).to_string()) + .collect(); + names.sort(); + names + } + + fn prop_repr(&self, prop_ids: &HashSet, meta: &PropMapper) -> String { + self.prop_names(prop_ids, meta) + .into_iter() + .map(|name| format!("('{}')", name)) + .collect::>() + .join(", ") + } + + fn node_const_meta(&self) -> &PropMapper { + self.graph.node_meta().const_prop_meta() + } + + fn node_temp_meta(&self) -> &PropMapper { + self.graph.node_meta().temporal_prop_meta() + } + + fn edge_const_meta(&self) -> &PropMapper { + self.graph.edge_meta().const_prop_meta() + } + + fn edge_temp_meta(&self) -> &PropMapper { + self.graph.edge_meta().temporal_prop_meta() + } +} + +#[pyclass(name = "IndexSpecBuilder", module = "raphtory")] +#[derive(Clone)] +pub struct PyIndexSpecBuilder { + builder: IndexSpecBuilder, +} + +#[pymethods] +impl PyIndexSpecBuilder { + #[new] + pub fn new(graph: MaterializedGraph) -> Self { + Self { + builder: IndexSpecBuilder::new(graph), + } + } + + pub fn with_all_node_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_node_props(), + }) + } + + pub fn with_all_const_node_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_const_node_props(), + }) + } + + pub fn with_all_temp_node_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_temp_node_props(), + }) + } + + pub fn with_const_node_props(&mut self, props: Vec) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_const_node_props(props)?, + }) + } + + pub fn with_temp_node_props(&mut self, props: Vec) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_temp_node_props(props)?, + }) + } + + pub fn with_all_edge_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_edge_props(), + }) + } + + pub fn with_all_edge_const_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_edge_const_props(), + }) + } + + pub fn with_all_temp_edge_props(&mut self) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_all_temp_edge_props(), + }) + } + + pub fn with_const_edge_props(&mut self, props: Vec) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_const_edge_props(props)?, + }) + } + + pub fn with_temp_edge_props(&mut self, props: Vec) -> PyResult { + Ok(Self { + builder: self.builder.clone().with_temp_edge_props(props)?, + }) + } + + pub fn build(&self) -> PyIndexSpec { + PyIndexSpec { + graph: self.builder.graph.clone().into_dynamic(), + spec: self.builder.build(), + } + } +} + +#[pymethods] +impl PyGraphView { + /// Get index spec + fn get_index_spec(&self) -> Result { + let spec = self.graph.get_index_spec()?; + Ok(PyIndexSpec { + graph: self.graph.clone(), + spec, + }) } /// Searches for nodes which match the given filter expression. This uses Tantivy's exact search. diff --git a/raphtory/src/python/packages/base_modules.rs b/raphtory/src/python/packages/base_modules.rs index d78c148d43..67f7889047 100644 --- a/raphtory/src/python/packages/base_modules.rs +++ b/raphtory/src/python/packages/base_modules.rs @@ -57,6 +57,8 @@ pub fn add_raphtory_classes(m: &Bound) -> PyResult<()> { PyPropertyRef, PyPropertyFilter, PyWindowSet, + PyIndexSpecBuilder, + PyIndexSpec ); #[cfg(feature = "storage")] @@ -152,6 +154,10 @@ pub fn base_vectors_module(py: Python<'_>) -> Result, PyErr> { pub use crate::python::graph::node_state::base_node_state_module; use crate::python::{ - algorithm::epidemics::PyInfected, graph::properties::PropertiesView, + algorithm::epidemics::PyInfected, + graph::{ + index::{PyIndexSpec, PyIndexSpecBuilder}, + properties::PropertiesView, + }, types::wrappers::document::PyEmbedding, }; diff --git a/raphtory/src/search/edge_filter_executor.rs b/raphtory/src/search/edge_filter_executor.rs index 6f87ee3862..d49ff34b42 100644 --- a/raphtory/src/search/edge_filter_executor.rs +++ b/raphtory/src/search/edge_filter_executor.rs @@ -107,7 +107,7 @@ impl<'a> EdgeFilterExecutor<'a> { match query { Some(query) => self.execute_filter_query(graph, query, &pi.reader, limit, offset), // Fallback to raphtory apis - None => Self::raph_filter_edges(graph, filter, offset, limit), + None => Self::raph_filter_edges(graph, filter, limit, offset), } } @@ -136,7 +136,7 @@ impl<'a> EdgeFilterExecutor<'a> { collector_fn, ), // Fallback to raphtory apis - None => Self::raph_filter_edges(graph, filter, offset, limit), + None => Self::raph_filter_edges(graph, filter, limit, offset), } } @@ -156,7 +156,7 @@ impl<'a> EdgeFilterExecutor<'a> { { self.execute_or_fallback(graph, &cpi, filter, limit, offset) } else { - Err(GraphError::PropertyNotFound(prop_name.to_string())) + Self::raph_filter_edges(graph, filter, limit, offset) } } @@ -188,7 +188,7 @@ impl<'a> EdgeFilterExecutor<'a> { collector_fn, ) } else { - Err(GraphError::PropertyNotFound(prop_name.to_string())) + Self::raph_filter_edges(graph, filter, limit, offset) } } @@ -248,7 +248,7 @@ impl<'a> EdgeFilterExecutor<'a> { offset, LatestEdgePropertyFilterCollector::new, ), - _ => Err(GraphError::PropertyNotFound(prop_name.to_string())), + _ => Self::raph_filter_edges(graph, filter, limit, offset), } } @@ -305,7 +305,7 @@ impl<'a> EdgeFilterExecutor<'a> { offset, )?, None => { - Self::raph_filter_edges(graph, &EdgeFieldFilter(filter.clone()), offset, limit)? + Self::raph_filter_edges(graph, &EdgeFieldFilter(filter.clone()), limit, offset)? } }; @@ -350,7 +350,7 @@ impl<'a> EdgeFilterExecutor<'a> { Ok(combined.into_iter().collect()) } - CompositeEdgeFilter::Not(_) => Self::raph_filter_edges(graph, filter, offset, limit), + CompositeEdgeFilter::Not(_) => Self::raph_filter_edges(graph, filter, limit, offset), } } @@ -410,8 +410,8 @@ impl<'a> EdgeFilterExecutor<'a> { fn raph_filter_edges( graph: &G, filter: &(impl InternalEdgeFilterOps + Clone), - offset: usize, limit: usize, + offset: usize, ) -> Result>, GraphError> { let filtered_edges = graph .filter_edges(filter.clone())? diff --git a/raphtory/src/search/edge_index.rs b/raphtory/src/search/edge_index.rs index 27607113b8..25655ea500 100644 --- a/raphtory/src/search/edge_index.rs +++ b/raphtory/src/search/edge_index.rs @@ -1,14 +1,9 @@ use crate::{ - core::{ - entities::{EID, VID}, - storage::timeindex::{AsTime, TimeIndexEntry}, - utils::errors::GraphError, - }, + core::{entities::EID, storage::timeindex::TimeIndexEntry, utils::errors::GraphError}, db::{ api::{ - properties::internal::{ConstPropertiesOps, TemporalPropertiesOps}, storage::graph::{edges::edge_storage_ops::EdgeStorageOps, storage_ops::GraphStorage}, - view::internal::core_ops::CoreGraphOps, + view::{internal::core_ops::CoreGraphOps, IndexSpec}, }, graph::edge::EdgeView, }, @@ -16,14 +11,15 @@ use crate::{ search::{ entity_index::EntityIndex, fields::{DESTINATION, DESTINATION_TOKENIZED, EDGE_ID, SOURCE, SOURCE_TOKENIZED}, - TOKENIZER, + resolve_props, TOKENIZER, }, }; use raphtory_api::core::storage::dict_mapper::MaybeNew; use rayon::prelude::ParallelIterator; use std::{ + collections::HashSet, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; use tantivy::{ collector::TopDocs, @@ -117,6 +113,16 @@ impl EdgeIndex { }) } + pub(crate) fn resolve_const_props(&self) -> HashSet { + let props = self.entity_index.const_property_indexes.read(); + resolve_props(&props) + } + + pub(crate) fn resolve_temp_props(&self) -> HashSet { + let props = self.entity_index.temporal_property_indexes.read(); + resolve_props(&props) + } + pub(crate) fn print(&self) -> Result<(), GraphError> { let searcher = self.entity_index.reader.searcher(); let top_docs = searcher.search(&AllQuery, &TopDocs::with_limit(1000))?; @@ -208,18 +214,13 @@ impl EdgeIndex { &self, edge_id: EID, layer_id: usize, - const_writers: &mut [Option], - const_props: &[(usize, Prop)], + writers: &mut [Option], + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let edge_id = edge_id.as_u64(); - self.entity_index.index_edge_const_properties( - edge_id, - layer_id, - const_writers, - const_props.iter().map(|(id, prop)| (*id, prop)), - )?; - - self.entity_index.commit_writers(const_writers) + self.entity_index + .index_edge_const_properties(edge_id, layer_id, writers, props)?; + self.entity_index.commit_writers(writers) } fn index_edge_t( @@ -230,7 +231,7 @@ impl EdgeIndex { layer_id: usize, writer: &mut IndexWriter, temporal_writers: &mut [Option], - temporal_props: &[(usize, Prop)], + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let eid_u64 = edge_id.inner().as_u64(); self.entity_index.index_edge_temporal_properties( @@ -238,7 +239,7 @@ impl EdgeIndex { eid_u64, layer_id, temporal_writers, - temporal_props.iter().map(|(id, prop)| (*id, prop)), + props, )?; // Check if the edge document is already in the index, @@ -283,7 +284,11 @@ impl EdgeIndex { edge_id, layer_id, const_writers, - edge.properties().constant().iter_id(), + &*edge + .properties() + .constant() + .iter_id() + .collect::>(), )?; for edge in edge.explode() { @@ -294,7 +299,7 @@ impl EdgeIndex { edge_id, layer_id, temporal_writers, - temporal_properties, + &*temporal_properties.collect::>(), )?; } } @@ -308,64 +313,49 @@ impl EdgeIndex { } pub(crate) fn index_edges( + &self, graph: &GraphStorage, - path: Option<&Path>, + path: Option, + index_spec: &IndexSpec, ) -> Result { - let edge_index_path = path.as_deref().map(|p| p.join("edges")); - let edge_index = EdgeIndex::new(&edge_index_path)?; - // Initialize property indexes and get their writers - let const_property_keys = graph.edge_meta().const_prop_meta().get_keys().into_iter(); - let const_properties_index_path = edge_index_path - .as_deref() - .map(|p| p.join("const_properties")); - let mut const_writers = edge_index - .entity_index - .initialize_edge_const_property_indexes( - graph, - const_property_keys, - &const_properties_index_path, - )?; + let const_properties_index_path = path.as_deref().map(|p| p.join("const_properties")); + let mut const_writers = self.entity_index.initialize_edge_const_property_indexes( + graph.edge_meta().const_prop_meta(), + &const_properties_index_path, + &index_spec.edge_const_props, + )?; - let temporal_property_keys = graph - .edge_meta() - .temporal_prop_meta() - .get_keys() - .into_iter(); - let temporal_properties_index_path = edge_index_path - .as_deref() - .map(|p| p.join("temporal_properties")); - let mut temporal_writers = edge_index + let temporal_properties_index_path = path.as_deref().map(|p| p.join("temporal_properties")); + let mut temporal_writers = self .entity_index .initialize_edge_temporal_property_indexes( - graph, - temporal_property_keys, + graph.edge_meta().temporal_prop_meta(), &temporal_properties_index_path, + &index_spec.edge_temp_props, )?; - let mut writer = edge_index.entity_index.index.writer(100_000_000)?; + let mut writer = self.entity_index.index.writer(100_000_000)?; let locked_g = graph.core_graph(); locked_g.edges_par(&graph).try_for_each(|e_ref| { { let e_view = EdgeView::new(graph.clone(), e_ref); - edge_index.index_edge(graph, e_view, &writer, &const_writers, &temporal_writers)?; + self.index_edge(graph, e_view, &writer, &const_writers, &temporal_writers)?; } Ok::<(), GraphError>(()) })?; // Commit writers - edge_index.entity_index.commit_writers(&mut const_writers)?; - edge_index - .entity_index - .commit_writers(&mut temporal_writers)?; + self.entity_index.commit_writers(&mut const_writers)?; + self.entity_index.commit_writers(&mut temporal_writers)?; writer.commit()?; // Reload readers - edge_index.entity_index.reload_const_property_indexes()?; - edge_index.entity_index.reload_temporal_property_indexes()?; - edge_index.entity_index.reader.reload()?; + self.entity_index.reload_const_property_indexes()?; + self.entity_index.reload_temporal_property_indexes()?; + self.entity_index.reader.reload()?; - Ok(edge_index) + Ok(self.clone()) } pub(crate) fn add_edge_update( @@ -373,20 +363,10 @@ impl EdgeIndex { graph: &GraphStorage, edge_id: MaybeNew, t: TimeIndexEntry, - src: VID, - dst: VID, layer_id: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let edge = graph - .edge(src, dst) - .expect("Edge for internal id should exist.") - .at(t.t()); - - let temporal_prop_ids = edge.temporal_prop_ids(); - let mut temporal_writers = self - .entity_index - .get_temporal_property_writers(temporal_prop_ids)?; + let mut temporal_writers = self.entity_index.get_temporal_property_writers(props)?; let mut writer = self.entity_index.index.writer(100_000_000)?; self.index_edge_t( @@ -399,6 +379,7 @@ impl EdgeIndex { props, )?; + self.entity_index.reload_temporal_property_indexes()?; self.entity_index.reader.reload()?; Ok(()) @@ -406,21 +387,11 @@ impl EdgeIndex { pub(crate) fn add_edge_constant_properties( &self, - graph: &GraphStorage, edge_id: EID, layer_id: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let src = graph.core_edge(edge_id).src(); - let dst = graph.core_edge(edge_id).dst(); - let edge = graph - .edge(src, dst) - .expect("Edge for internal id should exist."); - - let const_property_ids = edge.const_prop_ids(); - let mut const_writers = self - .entity_index - .get_const_property_writers(const_property_ids)?; + let mut const_writers = self.entity_index.get_const_property_writers(props)?; self.index_edge_c(edge_id, layer_id, &mut const_writers, props)?; @@ -431,27 +402,17 @@ impl EdgeIndex { pub(crate) fn update_edge_constant_properties( &self, - graph: &GraphStorage, edge_id: EID, layer_id: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let src = graph.core_edge(edge_id).src(); - let dst = graph.core_edge(edge_id).dst(); - let edge = graph - .edge(src, dst) - .expect("Edge for internal id should exist."); - - let const_property_ids = edge.const_prop_ids(); - let mut const_writers = self - .entity_index - .get_const_property_writers(const_property_ids)?; + let mut const_writers = self.entity_index.get_const_property_writers(props)?; // Delete existing constant property document self.entity_index.delete_const_properties_index_docs( edge_id.as_u64(), &mut const_writers, - props.iter().map(|(id, prop)| (*id, prop)), + props, )?; // Reindex the edge's constant properties diff --git a/raphtory/src/search/entity_index.rs b/raphtory/src/search/entity_index.rs index 553996f471..f948b516ba 100644 --- a/raphtory/src/search/entity_index.rs +++ b/raphtory/src/search/entity_index.rs @@ -1,16 +1,16 @@ use crate::{ core::{utils::errors::GraphError, Prop}, - db::api::storage::graph::storage_ops::GraphStorage, search::{fields, new_index, property_index::PropertyIndex, register_default_tokenizers}, }; use itertools::Itertools; -use parking_lot::RwLock; +use lock_api::RwLockReadGuard; +use parking_lot::{RawRwLock, RwLock}; use raphtory_api::core::{ entities::properties::props::{Meta, PropMapper}, - storage::{arc_str::ArcStr, dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, + storage::timeindex::TimeIndexEntry, PropType, }; -use std::{borrow::Borrow, path::PathBuf, sync::Arc}; +use std::{borrow::Borrow, collections::HashSet, path::PathBuf, sync::Arc}; use tantivy::{ schema::{Schema, SchemaBuilder, FAST, INDEXED, STORED}, Index, IndexReader, IndexWriter, Term, @@ -67,63 +67,24 @@ impl EntityIndex { EntityIndex::load_from_path(path, true) } - pub(crate) fn create_property_index( - &self, - prop_id: MaybeNew, - prop_name: &str, - prop_type: &PropType, - is_static: bool, - add_const_schema_fields: fn(&mut SchemaBuilder), - add_temporal_schema_fields: fn(&mut SchemaBuilder), - new_property: fn(Schema, path: &Option) -> Result, - path: &Option, - ) -> Result<(), GraphError> { - prop_id - .if_new(|prop_id| { - let mut prop_index_guard = if is_static { - self.const_property_indexes.write() - } else { - self.temporal_property_indexes.write() - }; - - // Resize the vector if needed - if prop_id >= prop_index_guard.len() { - prop_index_guard.resize(prop_id + 1, None); - } - - let mut schema_builder = - PropertyIndex::schema_builder(&*prop_name, prop_type.clone()); - - let path = if is_static { - add_const_schema_fields(&mut schema_builder); - path.as_deref().map(|p| p.join("const_properties")) - } else { - add_temporal_schema_fields(&mut schema_builder); - path.as_deref().map(|p| p.join("temporal_properties")) - }; - - let schema = schema_builder.build(); - let prop_index_path = path.map(|p| p.join(prop_id.to_string())); - let property_index = new_property(schema, &prop_index_path)?; - prop_index_guard[prop_id] = Some(property_index); - - Ok::<_, GraphError>(()) - }) - .transpose()?; - Ok(()) - } - fn get_property_writers( &self, - prop_ids: impl Iterator, + props: &[(usize, Prop)], property_indexes: &RwLock>>, ) -> Result>, GraphError> { - let prop_index_guard = property_indexes.read(); + let indexes = property_indexes.read(); + + // Filter prop_ids for which there is a property index + let prop_ids = props + .iter() + .map(|(id, _)| *id) + .filter(|id| indexes.get(*id).map_or(false, |entry| entry.is_some())) + .collect::>(); let mut writers = Vec::new(); - writers.resize_with(prop_index_guard.len(), || None); + writers.resize_with(indexes.len(), || None); for id in prop_ids { - let writer = prop_index_guard[id] + let writer = indexes[id] .as_ref() .map(|index| index.index.writer(50_000_000)) .transpose()?; @@ -135,16 +96,16 @@ impl EntityIndex { pub(crate) fn get_const_property_writers( &self, - prop_ids: impl Iterator, + props: &[(usize, Prop)], ) -> Result>, GraphError> { - self.get_property_writers(prop_ids, &self.const_property_indexes) + self.get_property_writers(props, &self.const_property_indexes) } pub(crate) fn get_temporal_property_writers( &self, - prop_ids: impl Iterator, + props: &[(usize, Prop)], ) -> Result>, GraphError> { - self.get_property_writers(prop_ids, &self.temporal_property_indexes) + self.get_property_writers(props, &self.temporal_property_indexes) } // We initialize the property indexes per property as and when we discover a new property while processing each node and edge update. @@ -152,45 +113,47 @@ impl EntityIndex { // which is why create all the property indexes upfront. fn initialize_property_indexes( &self, - graph: &GraphStorage, + meta: &PropMapper, property_indexes: &RwLock>>, - prop_keys: impl Iterator, - get_property_meta: fn(&GraphStorage) -> &PropMapper, add_schema_fields: fn(&mut SchemaBuilder), new_property: fn(Schema, &Option) -> Result, path: &Option, + props: &HashSet, ) -> Result>, GraphError> { - let prop_meta = get_property_meta(graph); - let properties = prop_keys - .filter_map(|k| { - prop_meta.get_id(&*k).and_then(|prop_id| { - prop_meta - .get_dtype(prop_id) - .map(|prop_type| (k.to_string(), prop_id, prop_type)) - }) + let mut indexes = property_indexes.write(); + let mut writers: Vec> = Vec::new(); + + let properties: Vec<(String, usize, PropType)> = props + .into_iter() + .filter_map(|prop_id| { + let prop_name = meta.get_name(*prop_id).to_string(); + meta.get_dtype(*prop_id) + .map(|prop_type| (prop_name, *prop_id, prop_type)) }) .collect_vec(); - let mut prop_index_guard = property_indexes.write(); - let mut writers: Vec> = Vec::new(); - for (prop_name, prop_id, prop_type) in properties { // Resize the vector if needed - if prop_id >= prop_index_guard.len() { - prop_index_guard.resize(prop_id + 1, None); + if prop_id >= indexes.len() { + indexes.resize(prop_id + 1, None); + } + // Resize the writers if needed + if prop_id >= writers.len() { + writers.resize_with(prop_id + 1, || None); } // Create a new PropertyIndex if it doesn't exist - if prop_index_guard[prop_id].is_none() { - let mut schema_builder = PropertyIndex::schema_builder(&*prop_name, prop_type); + if indexes[prop_id].is_none() { + let mut schema_builder = + PropertyIndex::schema_builder(&*prop_name, prop_type.clone()); add_schema_fields(&mut schema_builder); let schema = schema_builder.build(); let prop_index_path = path.as_deref().map(|p| p.join(prop_id.to_string())); let property_index = new_property(schema, &prop_index_path)?; let writer = property_index.index.writer(50_000_000)?; - writers.push(Some(writer)); - prop_index_guard[prop_id] = Some(property_index); + writers[prop_id] = Some(writer); + indexes[prop_id] = Some(property_index); } } @@ -199,34 +162,31 @@ impl EntityIndex { pub(crate) fn initialize_node_const_property_indexes( &self, - graph: &GraphStorage, - prop_keys: impl Iterator, + meta: &PropMapper, path: &Option, + node_const_props: &HashSet, ) -> Result>, GraphError> { self.initialize_property_indexes( - graph, + meta, &self.const_property_indexes, - prop_keys, - |g| g.node_meta().const_prop_meta(), |schema| { schema.add_u64_field(fields::NODE_ID, INDEXED | FAST | STORED); }, PropertyIndex::new_node_property, path, + node_const_props, ) } pub(crate) fn initialize_node_temporal_property_indexes( &self, - graph: &GraphStorage, - prop_keys: impl Iterator, + meta: &PropMapper, path: &Option, + node_temp_props: &HashSet, ) -> Result>, GraphError> { self.initialize_property_indexes( - graph, + meta, &self.temporal_property_indexes, - prop_keys, - |g| g.node_meta().temporal_prop_meta(), |schema| { schema.add_i64_field(fields::TIME, INDEXED | FAST | STORED); schema.add_u64_field(fields::SECONDARY_TIME, INDEXED | FAST | STORED); @@ -234,40 +194,38 @@ impl EntityIndex { }, PropertyIndex::new_node_property, path, + node_temp_props, ) } pub(crate) fn initialize_edge_const_property_indexes( &self, - graph: &GraphStorage, - prop_keys: impl Iterator, + meta: &PropMapper, path: &Option, + edge_const_props: &HashSet, ) -> Result>, GraphError> { self.initialize_property_indexes( - graph, + meta, &self.const_property_indexes, - prop_keys, - |g| g.edge_meta().const_prop_meta(), |schema| { schema.add_u64_field(fields::EDGE_ID, INDEXED | FAST | STORED); schema.add_u64_field(fields::LAYER_ID, INDEXED | FAST | STORED); }, PropertyIndex::new_edge_property, path, + edge_const_props, ) } pub(crate) fn initialize_edge_temporal_property_indexes( &self, - graph: &GraphStorage, - prop_keys: impl Iterator, + meta: &PropMapper, path: &Option, + edge_temp_props: &HashSet, ) -> Result>, GraphError> { self.initialize_property_indexes( - graph, + meta, &self.temporal_property_indexes, - prop_keys, - |g| g.edge_meta().temporal_prop_meta(), |schema| { schema.add_i64_field(fields::TIME, INDEXED | FAST | STORED); schema.add_u64_field(fields::SECONDARY_TIME, INDEXED | FAST | STORED); @@ -276,25 +234,37 @@ impl EntityIndex { }, PropertyIndex::new_edge_property, path, + edge_temp_props, ) } + // Filter props for which there already is a property index + fn filtered_props( + props: &[(usize, Prop)], + indexes: &RwLockReadGuard>>, + ) -> Vec<(usize, Prop)> { + props + .iter() + .cloned() + .filter(|(id, _)| indexes.get(*id).map_or(false, |entry| entry.is_some())) + .collect() + } + pub(crate) fn delete_const_properties_index_docs( &self, entity_id: u64, writers: &mut [Option], - props: impl Iterator)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let property_indexes = self.const_property_indexes.read(); - for (prop_id, _prop_value) in props { - if let Some(Some(prop_writer)) = writers.get(prop_id) { - if let Some(property_index) = &property_indexes[prop_id] { - let term = Term::from_field_u64(property_index.entity_id_field, entity_id); - prop_writer.delete_term(term); + let indexes = self.const_property_indexes.read(); + for (prop_id, _) in Self::filtered_props(props, &indexes) { + if let Some(Some(writer)) = writers.get(prop_id) { + if let Some(index) = &indexes[prop_id] { + let term = Term::from_field_u64(index.entity_id_field, entity_id); + writer.delete_term(term); } } } - self.commit_writers(writers)?; Ok(()) } @@ -303,19 +273,18 @@ impl EntityIndex { &self, node_id: u64, writers: &[Option], - props: impl Iterator)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let property_indexes = self.const_property_indexes.read(); - for (prop_id, prop_value) in props { - if let Some(Some(prop_writer)) = writers.get(prop_id) { - if let Some(property_index) = &property_indexes[prop_id] { - let prop_doc = property_index - .create_node_const_property_document(node_id, prop_value.borrow())?; - prop_writer.add_document(prop_doc)?; + let indexes = self.const_property_indexes.read(); + for (prop_id, prop_value) in Self::filtered_props(props, &indexes) { + if let Some(Some(writer)) = writers.get(prop_id) { + if let Some(index) = &indexes[prop_id] { + let prop_doc = + index.create_node_const_property_document(node_id, prop_value.borrow())?; + writer.add_document(prop_doc)?; } } } - Ok(()) } @@ -324,22 +293,21 @@ impl EntityIndex { time: TimeIndexEntry, node_id: u64, writers: &[Option], - props: impl IntoIterator)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let property_indexes = self.temporal_property_indexes.read(); - for (prop_id, prop) in props { - if let Some(Some(prop_writer)) = writers.get(prop_id) { - if let Some(property_index) = &property_indexes[prop_id] { - let prop_doc = property_index.create_node_temporal_property_document( + let indexes = self.temporal_property_indexes.read(); + for (prop_id, prop) in Self::filtered_props(props, &indexes) { + if let Some(Some(writer)) = writers.get(prop_id) { + if let Some(index) = &indexes[prop_id] { + let prop_doc = index.create_node_temporal_property_document( time, node_id, prop.borrow(), )?; - prop_writer.add_document(prop_doc)?; + writer.add_document(prop_doc)?; } } } - Ok(()) } @@ -348,22 +316,21 @@ impl EntityIndex { edge_id: u64, layer_id: usize, writers: &[Option], - props: impl Iterator)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let property_indexes = self.const_property_indexes.read(); - for (prop_id, prop_value) in props { - if let Some(Some(prop_writer)) = writers.get(prop_id) { - if let Some(property_index) = &property_indexes[prop_id] { - let prop_doc = property_index.create_edge_const_property_document( + let indexes = self.const_property_indexes.read(); + for (prop_id, prop_value) in Self::filtered_props(props, &indexes) { + if let Some(Some(writer)) = writers.get(prop_id) { + if let Some(index) = &indexes[prop_id] { + let prop_doc = index.create_edge_const_property_document( edge_id, layer_id, prop_value.borrow(), )?; - prop_writer.add_document(prop_doc)?; + writer.add_document(prop_doc)?; } } } - Ok(()) } @@ -373,23 +340,22 @@ impl EntityIndex { edge_id: u64, layer_id: usize, writers: &[Option], - props: impl Iterator)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let property_indexes = self.temporal_property_indexes.read(); - for (prop_id, prop) in props { - if let Some(Some(prop_writer)) = writers.get(prop_id) { - if let Some(property_index) = &property_indexes[prop_id] { - let prop_doc = property_index.create_edge_temporal_property_document( + let indexes = self.temporal_property_indexes.read(); + for (prop_id, prop) in Self::filtered_props(props, &indexes) { + if let Some(Some(writer)) = writers.get(prop_id) { + if let Some(index) = &indexes[prop_id] { + let prop_doc = index.create_edge_temporal_property_document( time, edge_id, layer_id, prop.borrow(), )?; - prop_writer.add_document(prop_doc)?; + writer.add_document(prop_doc)?; } } } - Ok(()) } @@ -435,26 +401,26 @@ impl EntityIndex { &self, writers: &mut [Option], ) -> Result<(), GraphError> { - for writer_option in writers { - if let Some(const_writer) = writer_option { - const_writer.commit()?; + for writer in writers { + if let Some(writer) = writer { + writer.commit()?; } } Ok(()) } pub(crate) fn reload_const_property_indexes(&self) -> Result<(), GraphError> { - let const_indexes = self.const_property_indexes.read(); - for property_index_option in const_indexes.iter().flatten() { - property_index_option.reader.reload()?; + let indexes = self.const_property_indexes.read(); + for index in indexes.iter().flatten() { + index.reader.reload()?; } Ok(()) } pub(crate) fn reload_temporal_property_indexes(&self) -> Result<(), GraphError> { - let temporal_indexes = self.temporal_property_indexes.read(); - for property_index_option in temporal_indexes.iter().flatten() { - property_index_option.reader.reload()?; + let indexes = self.temporal_property_indexes.read(); + for index in indexes.iter().flatten() { + index.reader.reload()?; } Ok(()) } diff --git a/raphtory/src/search/graph_index.rs b/raphtory/src/search/graph_index.rs index cabdee1a7c..023ea86cf3 100644 --- a/raphtory/src/search/graph_index.rs +++ b/raphtory/src/search/graph_index.rs @@ -4,14 +4,12 @@ use crate::{ storage::timeindex::TimeIndexEntry, utils::errors::GraphError, }, - db::api::storage::graph::storage_ops::GraphStorage, + db::api::{storage::graph::storage_ops::GraphStorage, view::IndexSpec}, prelude::*, - search::{ - edge_index::EdgeIndex, fields, node_index::NodeIndex, property_index::PropertyIndex, - searcher::Searcher, - }, + search::{edge_index::EdgeIndex, node_index::NodeIndex, searcher::Searcher}, }; -use raphtory_api::core::{storage::dict_mapper::MaybeNew, PropType}; +use parking_lot::RwLock; +use raphtory_api::core::storage::dict_mapper::MaybeNew; use std::{ ffi::OsStr, fmt::{Debug, Formatter}, @@ -20,7 +18,6 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tantivy::schema::{FAST, INDEXED, STORED}; use tempfile::TempDir; use uuid::Uuid; use walkdir::WalkDir; @@ -31,6 +28,7 @@ pub struct GraphIndex { pub(crate) node_index: NodeIndex, pub(crate) edge_index: EdgeIndex, pub path: Option>, // If path is None, index is created in-memory + pub index_spec: Arc>, } impl Debug for GraphIndex { @@ -38,6 +36,8 @@ impl Debug for GraphIndex { f.debug_struct("GraphIndex") .field("node_index", &self.node_index) .field("edge_index", &self.edge_index) + .field("path", &self.path.as_ref().map(|p| p.path())) + .field("index_spec", &self.index_spec) .finish() } } @@ -142,20 +142,37 @@ impl GraphIndex { let edge_index = EdgeIndex::load_from_path(&tmp_path.path().join("edges"))?; let path = Some(Arc::new(tmp_path)); + let index_spec = IndexSpec { + node_const_props: node_index.resolve_const_props(), + node_temp_props: node_index.resolve_temp_props(), + edge_const_props: edge_index.resolve_const_props(), + edge_temp_props: edge_index.resolve_temp_props(), + }; + Ok(GraphIndex { - node_index, - edge_index, + node_index: node_index.clone(), + edge_index: edge_index.clone(), path, + index_spec: Arc::new(RwLock::new(index_spec)), }) } - pub fn create_from_graph( + fn get_node_index_path(path: &Option>) -> Option { + path.as_ref().map(|d| d.path().join("nodes")) + } + + fn get_edge_index_path(path: &Option>) -> Option { + path.as_ref().map(|d| d.path().join("edges")) + } + + pub fn create( graph: &GraphStorage, create_in_ram: bool, - cache_path: Option<&Path>, + cached_graph_path: Option<&Path>, + index_spec: IndexSpec, ) -> Result { let dir = if !create_in_ram { - let temp_dir = match cache_path { + let temp_dir = match cached_graph_path { // Creates index in a temp dir within cache graph dir. // The intention is to avoid creating index in a tmp dir that could be on another file system. Some(path) => TempDir::new_in(path)?, @@ -166,20 +183,40 @@ impl GraphIndex { None }; - let path = dir.as_ref().map(|p| p.path()); - let node_index = NodeIndex::index_nodes(graph, path)?; - // node_index.print()?; + let path = GraphIndex::get_node_index_path(&dir); + let node_index = NodeIndex::new(&path)?; + node_index.index_nodes(graph, path, &index_spec)?; - let edge_index = EdgeIndex::index_edges(graph, path)?; - // edge_index.print()?; + let path = GraphIndex::get_edge_index_path(&dir); + let edge_index = EdgeIndex::new(&path)?; + edge_index.index_edges(graph, path, &index_spec)?; Ok(GraphIndex { node_index, edge_index, path: dir, + index_spec: Arc::new(RwLock::new(index_spec)), }) } + pub fn update(&self, graph: &GraphStorage, index_spec: IndexSpec) -> Result<(), GraphError> { + let mut existing_spec = self.index_spec.write(); + + if let Some(diff_spec) = IndexSpec::diff(&*existing_spec, &index_spec) { + let path = GraphIndex::get_node_index_path(&self.path); + self.node_index.index_nodes(graph, path, &diff_spec)?; + // self.node_index.print()?; + + let path = GraphIndex::get_edge_index_path(&self.path); + self.edge_index.index_edges(graph, path, &diff_spec)?; + // self.edge_index.print()?; + + *existing_spec = IndexSpec::union(&*existing_spec, &diff_spec); + } + + Ok(()) + } + pub fn searcher(&self) -> Searcher { Searcher::new(self) } @@ -270,27 +307,27 @@ impl GraphIndex { v: MaybeNew, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - self.node_index.add_node_update(graph, t, v, props) + self.node_index.add_node_update(graph, t, v, props)?; + Ok(()) } pub(crate) fn add_node_constant_properties( &self, - graph: &GraphStorage, node_id: VID, props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.node_index - .add_node_constant_properties(graph, node_id, props) + .add_node_constant_properties(node_id, props)?; + Ok(()) } pub(crate) fn update_node_constant_properties( &self, - graph: &GraphStorage, node_id: VID, props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.node_index - .update_node_constant_properties(graph, node_id, props) + .update_node_constant_properties(node_id, props) } pub(crate) fn add_edge_update( @@ -298,89 +335,31 @@ impl GraphIndex { graph: &GraphStorage, edge_id: MaybeNew, t: TimeIndexEntry, - src: VID, - dst: VID, layer: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.edge_index - .add_edge_update(graph, edge_id, t, src, dst, layer, props) + .add_edge_update(graph, edge_id, t, layer, props) } pub(crate) fn add_edge_constant_properties( &self, - graph: &GraphStorage, edge_id: EID, layer: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.edge_index - .add_edge_constant_properties(graph, edge_id, layer, props) + .add_edge_constant_properties(edge_id, layer, props) } pub(crate) fn update_edge_constant_properties( &self, - graph: &GraphStorage, edge_id: EID, layer: usize, props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.edge_index - .update_edge_constant_properties(graph, edge_id, layer, props) - } - - pub(crate) fn create_edge_property_index( - &self, - prop_id: MaybeNew, - prop_name: &str, - prop_type: &PropType, - is_static: bool, // Const or Temporal Property - ) -> Result<(), GraphError> { - let edge_index_path = self.path.as_deref().map(|p| p.path().join("edges")); - self.edge_index.entity_index.create_property_index( - prop_id, - prop_name, - prop_type, - is_static, - |schema| { - schema.add_u64_field(fields::EDGE_ID, INDEXED | FAST | STORED); - schema.add_u64_field(fields::LAYER_ID, INDEXED | FAST | STORED); - }, - |schema| { - schema.add_i64_field(fields::TIME, INDEXED | FAST | STORED); - schema.add_u64_field(fields::SECONDARY_TIME, INDEXED | FAST | STORED); - schema.add_u64_field(fields::EDGE_ID, INDEXED | FAST | STORED); - schema.add_u64_field(fields::LAYER_ID, INDEXED | FAST | STORED); - }, - PropertyIndex::new_edge_property, - &edge_index_path, - ) - } - - pub(crate) fn create_node_property_index( - &self, - prop_id: MaybeNew, - prop_name: &str, - prop_type: &PropType, - is_static: bool, // Const or Temporal Property - ) -> Result<(), GraphError> { - let node_index_path = self.path.as_deref().map(|p| p.path().join("nodes")); - self.node_index.entity_index.create_property_index( - prop_id, - prop_name, - prop_type, - is_static, - |schema| { - schema.add_u64_field(fields::NODE_ID, INDEXED | FAST | STORED); - }, - |schema| { - schema.add_i64_field(fields::TIME, INDEXED | FAST | STORED); - schema.add_u64_field(fields::SECONDARY_TIME, INDEXED | FAST | STORED); - schema.add_u64_field(fields::NODE_ID, INDEXED | FAST | STORED); - }, - PropertyIndex::new_node_property, - &node_index_path, - ) + .update_edge_constant_properties(edge_id, layer, props) } } @@ -391,6 +370,10 @@ mod graph_index_test { prelude::{AdditionOps, EdgeViewOps, Graph, GraphViewOps, NodeViewOps, PropertyFilter}, }; + #[cfg(feature = "search")] + use crate::db::graph::assertions::{search_edges, search_nodes}; + use crate::prelude::IndexMutationOps; + fn init_nodes_graph(graph: Graph) -> Graph { graph .add_node(1, 1, [("p1", 1), ("p2", 2)], Some("fire_nation")) @@ -409,16 +392,12 @@ mod graph_index_test { fn init_edges_graph(graph: Graph) -> Graph { graph - .add_edge(1, 1, 2, [("p1", 1), ("p2", 2)], Some("fire_nation")) - .unwrap(); - graph - .add_edge(2, 1, 2, [("p6", 6)], Some("fire_nation")) - .unwrap(); - graph - .add_edge(2, 2, 3, [("p4", 5)], Some("fire_nation")) + .add_edge(1, 1, 2, [("p1", 1), ("p2", 2)], None) .unwrap(); + graph.add_edge(2, 1, 2, [("p6", 6)], None).unwrap(); + graph.add_edge(2, 2, 3, [("p4", 5)], None).unwrap(); graph - .add_edge(3, 3, 4, [("p2", 4), ("p3", 3)], Some("water_tribe")) + .add_edge(3, 3, 4, [("p2", 4), ("p3", 3)], None) .unwrap(); graph } @@ -466,9 +445,7 @@ mod graph_index_test { .unwrap(); let filter = PropertyFilter::property("x").constant().eq(1u64); - let res = graph.search_nodes(filter, 20, 0).unwrap(); - let res = res.iter().map(|n| n.name()).collect::>(); - assert_eq!(res, vec!["1"]); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); graph .node(1) @@ -476,9 +453,7 @@ mod graph_index_test { .update_constant_properties([("x", 2u64)]) .unwrap(); let filter = PropertyFilter::property("x").constant().eq(1u64); - let res = graph.search_nodes(filter, 20, 0).unwrap(); - let res = res.iter().map(|n| n.name()).collect::>(); - assert_eq!(res, Vec::<&str>::new()); + assert_eq!(search_nodes(&graph, filter.clone()), Vec::<&str>::new()); graph .node(1) @@ -486,9 +461,7 @@ mod graph_index_test { .update_constant_properties([("x", 2u64)]) .unwrap(); let filter = PropertyFilter::property("x").constant().eq(2u64); - let res = graph.search_nodes(filter, 20, 0).unwrap(); - let res = res.iter().map(|n| n.name()).collect::>(); - assert_eq!(res, vec!["1"]); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); } #[test] @@ -499,41 +472,26 @@ mod graph_index_test { graph .edge(1, 2) .unwrap() - .add_constant_properties([("x", 1u64)], Some("fire_nation")) + .add_constant_properties([("x", 1u64)], None) .unwrap(); let filter = PropertyFilter::property("x").constant().eq(1u64); - let res = graph.search_edges(filter, 20, 0).unwrap(); - let res = res - .iter() - .map(|e| format!("{}->{}", e.src().name(), e.dst().name())) - .collect::>(); - assert_eq!(res, vec!["1->2"]); + assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); graph .edge(1, 2) .unwrap() - .update_constant_properties([("x", 2u64)], Some("fire_nation")) + .update_constant_properties([("x", 2u64)], None) .unwrap(); let filter = PropertyFilter::property("x").constant().eq(1u64); - let res = graph.search_edges(filter, 20, 0).unwrap(); - let res = res - .iter() - .map(|e| format!("{}->{}", e.src().name(), e.dst().name())) - .collect::>(); - assert_eq!(res, Vec::<&str>::new()); + assert_eq!(search_edges(&graph, filter.clone()), Vec::<&str>::new()); graph .edge(1, 2) .unwrap() - .update_constant_properties([("x", 2u64)], Some("fire_nation")) + .update_constant_properties([("x", 2u64)], None) .unwrap(); let filter = PropertyFilter::property("x").constant().eq(2u64); - let res = graph.search_edges(filter, 20, 0).unwrap(); - let res = res - .iter() - .map(|e| format!("{}->{}", e.src().name(), e.dst().name())) - .collect::>(); - assert_eq!(res, vec!["1->2"]); + assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); } } diff --git a/raphtory/src/search/mod.rs b/raphtory/src/search/mod.rs index 26c12f3810..4ec76e2055 100644 --- a/raphtory/src/search/mod.rs +++ b/raphtory/src/search/mod.rs @@ -1,5 +1,5 @@ -use crate::core::utils::errors::GraphError; -use std::{fs::create_dir_all, path::PathBuf}; +use crate::{core::utils::errors::GraphError, search::property_index::PropertyIndex}; +use std::{collections::HashSet, fs::create_dir_all, path::PathBuf}; use tantivy::{ schema::Schema, tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer}, @@ -78,3 +78,747 @@ pub(crate) fn new_index( Ok((index, reader)) } + +fn resolve_props(props: &Vec>) -> HashSet { + props + .iter() + .enumerate() + .filter_map(|(idx, opt)| opt.as_ref().map(|_| idx)) + .collect() +} + +#[cfg(test)] +mod test_index { + #[cfg(feature = "search")] + mod test_index_io { + use crate::{ + core::{utils::errors::GraphError, Prop}, + db::{ + api::{ + mutation::internal::{InternalAdditionOps, InternalPropertyAdditionOps}, + view::{internal::InternalStorageOps, StaticGraphViewOps}, + }, + graph::{ + assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestVariants, + }, + views::filter::model::{AsNodeFilter, NodeFilter, NodeFilterBuilderOps}, + }, + }, + prelude::{ + AdditionOps, CacheOps, Graph, GraphViewOps, IndexMutationOps, NodeViewOps, + PropertyAdditionOps, PropertyFilter, SearchableGraphOps, StableDecode, + StableEncode, + }, + serialise::GraphFolder, + }; + use raphtory_api::core::{storage::arc_str::ArcStr, utils::logging::global_info_logger}; + + fn init_graph(graph: G) -> G + where + G: StaticGraphViewOps + + AdditionOps + + InternalAdditionOps + + InternalPropertyAdditionOps + + PropertyAdditionOps, + { + graph + .add_node( + 1, + "Alice", + vec![("p1", Prop::U64(2u64))], + Some("fire_nation"), + ) + .unwrap(); + graph + } + + fn assert_search_results( + graph: &Graph, + filter: &T, + expected: Vec<&str>, + ) { + let res = graph + .search_nodes(filter.clone(), 2, 0) + .unwrap() + .into_iter() + .map(|n| n.name()) + .collect::>(); + assert_eq!(res, expected); + } + + #[test] + fn test_create_no_index_persist_no_index_on_encode_load_no_index_on_decode() { + // No index persisted since it was never created + let graph = init_graph(Graph::new()); + + let err = graph + .search_nodes(NodeFilter::name().eq("Alice"), 2, 0) + .expect_err("Expected error since index was not created"); + assert!(matches!(err, GraphError::IndexNotCreated)); + + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let index = graph.get_storage().unwrap().index.get(); + assert!(index.is_none()); + } + + #[test] + fn test_create_index_persist_index_on_encode_load_index_on_decode() { + let graph = init_graph(Graph::new()); + + // Created index + graph.create_index().unwrap(); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + + // Persisted both graph and index + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Loaded index that was persisted + let graph = Graph::decode(path).unwrap(); + let index = graph.get_storage().unwrap().index.get(); + assert!(index.is_some()); + + assert_search_results(&graph, &filter, vec!["Alice"]); + } + + #[test] + fn test_create_index_persist_index_on_encode_update_index_load_persisted_index_on_decode() { + let graph = init_graph(Graph::new()); + + // Created index + graph.create_index().unwrap(); + + let filter1 = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter1, vec!["Alice"]); + + // Persisted both graph and index + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Updated both graph and index + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + ) + .unwrap(); + let filter2 = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + + // Loaded index that was persisted + let graph = Graph::decode(path).unwrap(); + let index = graph.get_storage().unwrap().index.get(); + assert!(index.is_some()); + assert_search_results(&graph, &filter1, vec!["Alice"]); + assert_search_results(&graph, &filter2, Vec::<&str>::new()); + + // Updating and encode the graph and index should decode the updated the graph as well as index + // So far we have the index that was created and persisted for the first time + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + ) + .unwrap(); + let filter2 = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + + // Should persist the updated graph and index + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Should load the updated graph and index + let graph = Graph::decode(path).unwrap(); + let index = graph.get_storage().unwrap().index.get(); + assert!(index.is_some()); + assert_search_results(&graph, &filter1, vec!["Alice"]); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + } + + #[test] + fn test_zip_encode_decode_index() { + let graph = init_graph(Graph::new()); + graph.create_index().unwrap(); + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + let folder = GraphFolder::new_as_zip(path); + graph.encode(folder.root_folder).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let node = graph.node("Alice").unwrap(); + let node_type = node.node_type(); + assert_eq!(node_type, Some(ArcStr::from("fire_nation"))); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + } + + #[test] + fn test_create_index_in_ram() { + global_info_logger(); + + let graph = init_graph(Graph::new()); + graph.create_index_in_ram().unwrap(); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let index = graph.get_storage().unwrap().index.get(); + assert!(index.is_none()); + + let results = graph.search_nodes(filter.clone(), 2, 0); + assert!(matches!(results, Err(GraphError::IndexNotCreated))); + } + + #[test] + fn test_cached_graph_view() { + global_info_logger(); + let graph = init_graph(Graph::new()); + graph.create_index().unwrap(); + + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.cache(path).unwrap(); + + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + ) + .unwrap(); + graph.write_updates().unwrap(); + + let graph = Graph::decode(path).unwrap(); + let filter = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter, vec!["Tommy"]); + } + + #[test] + fn test_cached_graph_view_create_index_after_graph_is_cached() { + global_info_logger(); + let graph = init_graph(Graph::new()); + + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + graph.cache(path).unwrap(); + // Creates index in a temp dir within graph dir + graph.create_index().unwrap(); + + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + ) + .unwrap(); + graph.write_updates().unwrap(); + + let graph = Graph::decode(path).unwrap(); + let filter = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter, vec!["Tommy"]); + } + } + + mod test_index_spec { + #[cfg(feature = "search")] + use crate::prelude::SearchableGraphOps; + use crate::{ + core::utils::errors::GraphError, + db::{ + api::view::{internal::CoreGraphOps, IndexSpec, IndexSpecBuilder}, + graph::{ + assertions::{search_edges, search_nodes}, + views::filter::model::{ComposableFilter, PropertyFilterOps}, + }, + }, + prelude::{ + AdditionOps, EdgeViewOps, Graph, GraphViewOps, IndexMutationOps, NodeViewOps, + PropertyAdditionOps, PropertyFilter, StableDecode, + }, + serialise::{GraphFolder, StableEncode}, + }; + use ahash::HashSet; + use itertools::Itertools; + use raphtory_api::core::{entities::properties::props::PropMapper, PropType}; + + fn init_graph(mut graph: Graph) -> Graph { + let nodes = vec![ + ( + 1, + "pometry", + [("p1", 5u64), ("p2", 50u64)], + Some("fire_nation"), + [("x", true)], + ), + ( + 1, + "raphtory", + [("p1", 10u64), ("p2", 100u64)], + Some("water_tribe"), + [("y", false)], + ), + ]; + + for (time, name, props, group, const_props) in nodes { + let node = graph.add_node(time, name, props, group).unwrap(); + node.add_constant_properties(const_props).unwrap(); + } + + let edges = vec![ + ( + 1, + "pometry", + "raphtory", + [("e_p1", 3.2f64), ("e_p2", 10f64)], + None, + [("e_x", true)], + ), + ( + 1, + "raphtory", + "pometry", + [("e_p1", 4.0f64), ("e_p2", 20f64)], + None, + [("e_y", false)], + ), + ]; + + for (time, src, dst, props, label, const_props) in edges { + let edge = graph.add_edge(time, src, dst, props, label).unwrap(); + edge.add_constant_properties(const_props, label).unwrap(); + } + + graph + } + + fn props(graph: &Graph, index_spec: &IndexSpec) -> Vec> { + let node_const_prop_meta = graph.node_meta().const_prop_meta(); + let node_temp_prop_meta = graph.node_meta().temporal_prop_meta(); + let edge_const_prop_meta = graph.edge_meta().const_prop_meta(); + let edge_temp_prop_meta = graph.edge_meta().temporal_prop_meta(); + let extract_names = |props: &std::collections::HashSet, meta: &PropMapper| { + let mut names: Vec = props + .iter() + .map(|prop_id| meta.get_name(*prop_id).to_string()) + .collect(); + names.sort(); + names + }; + + vec![ + extract_names( + &index_spec.node_const_props, + graph.node_meta().const_prop_meta(), + ), + extract_names( + &index_spec.node_temp_props, + graph.node_meta().temporal_prop_meta(), + ), + extract_names( + &index_spec.edge_const_props, + graph.edge_meta().const_prop_meta(), + ), + extract_names( + &index_spec.edge_temp_props, + graph.edge_meta().temporal_prop_meta(), + ), + ] + } + + #[test] + fn test_with_all_props_index_spec() { + let graph = init_graph(Graph::new()); + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_all_node_props() + .with_all_edge_props() + .build(); + assert_eq!( + props(&graph, &index_spec), + vec![ + vec!["x", "y"], + vec!["p1", "p2"], + vec!["e_x", "e_y"], + vec!["e_p1", "e_p2"] + ] + ); + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = PropertyFilter::property("p1") + .eq(5u64) + .and(PropertyFilter::property("x").eq(true)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry"]); + + let filter = PropertyFilter::property("e_p1") + .lt(5f64) + .and(PropertyFilter::property("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + fn test_with_selected_props_index_spec() { + let graph = init_graph(Graph::new()); + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p1"]) + .unwrap() + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .with_temp_edge_props(vec!["e_p1"]) + .unwrap() + .build(); + assert_eq!( + props(&graph, &index_spec), + vec![vec!["y"], vec!["p1"], vec!["e_y"], vec!["e_p1"]] + ); + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = PropertyFilter::property("p1") + .eq(5u64) + .or(PropertyFilter::property("y").eq(false)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry", "raphtory"]); + + let filter = PropertyFilter::property("y").eq(false); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["raphtory"]); + + let filter = PropertyFilter::property("e_p1") + .lt(5f64) + .or(PropertyFilter::property("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + fn test_with_invalid_property_returns_error() { + let graph = init_graph(Graph::new()); + let result = IndexSpecBuilder::new(graph.clone()).with_const_node_props(["xyz"]); + + assert!(matches!(result, Err(GraphError::PropertyMissingError(p)) if p == "xyz")); + } + + #[test] + fn test_build_empty_spec_by_default() { + let graph = init_graph(Graph::new()); + let index_spec = IndexSpecBuilder::new(graph.clone()).build(); + + assert!(index_spec.node_const_props.is_empty()); + assert!(index_spec.node_temp_props.is_empty()); + assert!(index_spec.edge_const_props.is_empty()); + assert!(index_spec.edge_temp_props.is_empty()); + + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = PropertyFilter::property("p1") + .eq(5u64) + .and(PropertyFilter::property("x").eq(true)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry"]); + + let filter = PropertyFilter::property("e_p1") + .lt(5f64) + .or(PropertyFilter::property("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + fn test_mixed_node_and_edge_props_index_spec() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["x"]) + .unwrap() + .with_all_temp_node_props() + .with_all_edge_props() + .build(); + assert_eq!( + props(&graph, &index_spec), + vec![ + vec!["x"], + vec!["p1", "p2"], + vec!["e_x", "e_y"], + vec!["e_p1", "e_p2"] + ] + ); + + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = PropertyFilter::property("p1") + .eq(5u64) + .or(PropertyFilter::property("y").eq(false)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry", "raphtory"]); + + let filter = PropertyFilter::property("e_p1") + .lt(5f64) + .or(PropertyFilter::property("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + fn test_get_index_spec_newly_created_index() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["x"]) + .unwrap() + .with_all_temp_node_props() + .with_all_edge_props() + .build(); + + graph + .create_index_in_ram_with_spec(index_spec.clone()) + .unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + } + + #[test] + fn test_get_index_spec_updated_index() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, PropertyFilter::property("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, PropertyFilter::property("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p2"]) + .unwrap() + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, PropertyFilter::property("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, PropertyFilter::property("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + fn test_get_index_spec_updated_index_persisted_and_loaded() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + let tmp_graph_dir = tempfile::tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(path.clone()).unwrap(); + let graph = Graph::decode(path.clone()).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, PropertyFilter::property("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, PropertyFilter::property("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p2"]) + .unwrap() + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + let tmp_graph_dir = tempfile::tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(path.clone()).unwrap(); + let graph = Graph::decode(path).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, PropertyFilter::property("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, PropertyFilter::property("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + fn test_get_index_spec_loaded_index() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p2"]) + .unwrap() + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .with_temp_edge_props(vec!["e_p2"]) + .unwrap() + .build(); + + graph.create_index_with_spec(index_spec.clone()).unwrap(); + let tmp_graph_dir = tempfile::tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(path.clone()).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let index_spec2 = graph.get_index_spec().unwrap(); + + assert_eq!(index_spec, index_spec2); + } + + #[test] + fn test_get_index_spec_loaded_index_zip() { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p2"]) + .unwrap() + .with_const_edge_props(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + let binding = tempfile::TempDir::new().unwrap(); + let path = binding.path(); + let folder = GraphFolder::new_as_zip(path); + graph.encode(folder.root_folder).unwrap(); + + let graph = Graph::decode(path).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + } + + #[test] + fn test_no_new_node_prop_index_created_via_update_apis() { + run_node_index_test(|graph, index_spec| { + graph.create_index_with_spec(index_spec.clone()) + }); + + run_node_index_test(|graph, index_spec| { + graph.create_index_in_ram_with_spec(index_spec.clone()) + }); + } + + #[test] + fn test_no_new_edge_prop_index_created_via_update_apis() { + run_edge_index_test(|graph, index_spec| { + graph.create_index_with_spec(index_spec.clone()) + }); + + run_edge_index_test(|graph, index_spec| { + graph.create_index_in_ram_with_spec(index_spec.clone()) + }); + } + + fn run_node_index_test(create_index_fn: F) + where + F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, + { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p1"]) + .unwrap() + .build(); + create_index_fn(&graph, index_spec.clone()).unwrap(); + + let filter = PropertyFilter::property("p2").temporal().latest().eq(50u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["pometry"]); + + let node = graph + .add_node(1, "shivam", [("p1", 100u64)], Some("fire_nation")) + .unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + + let filter = PropertyFilter::property("p1") + .temporal() + .latest() + .eq(100u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + + node.add_constant_properties([("z", true)]).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = PropertyFilter::property("z").constant().eq(true); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + + node.update_constant_properties([("z", false)]).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = PropertyFilter::property("z").constant().eq(false); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + } + + fn run_edge_index_test(create_index_fn: F) + where + F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, + { + let graph = init_graph(Graph::new()); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_const_node_props(vec!["y"]) + .unwrap() + .with_temp_node_props(vec!["p2"]) + .unwrap() + .build(); + create_index_fn(&graph, index_spec.clone()).unwrap(); + + let edge = graph + .add_edge(1, "shivam", "kapoor", [("p1", 100u64)], None) + .unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = PropertyFilter::property("p1") + .temporal() + .latest() + .eq(100u64); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + + edge.add_constant_properties([("z", true)], None).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = PropertyFilter::property("z").constant().eq(true); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + + edge.update_constant_properties([("z", false)], None) + .unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = PropertyFilter::property("z").constant().eq(false); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + } + } +} diff --git a/raphtory/src/search/node_filter_executor.rs b/raphtory/src/search/node_filter_executor.rs index 364013bdb7..cbc4f0d9b1 100644 --- a/raphtory/src/search/node_filter_executor.rs +++ b/raphtory/src/search/node_filter_executor.rs @@ -107,7 +107,7 @@ impl<'a> NodeFilterExecutor<'a> { match query { Some(query) => self.execute_filter_query(graph, query, &pi.reader, limit, offset), // Fallback to raphtory apis - None => Self::raph_filter_nodes(graph, filter, offset, limit), + None => Self::raph_filter_nodes(graph, filter, limit, offset), } } @@ -156,7 +156,7 @@ impl<'a> NodeFilterExecutor<'a> { { self.execute_or_fallback(graph, &cpi, filter, limit, offset) } else { - Err(GraphError::PropertyNotFound(prop_name.to_string())) + Self::raph_filter_nodes(graph, filter, limit, offset) } } @@ -188,7 +188,7 @@ impl<'a> NodeFilterExecutor<'a> { collector_fn, ) } else { - Err(GraphError::PropertyNotFound(prop_name.to_string())) + Self::raph_filter_nodes(graph, filter, limit, offset) } } @@ -267,7 +267,7 @@ impl<'a> NodeFilterExecutor<'a> { offset, LatestNodePropertyFilterCollector::new, ), - _ => Err(GraphError::PropertyNotFound(prop_name.to_string())), + _ => Self::raph_filter_nodes(graph, filter, limit, offset), } } diff --git a/raphtory/src/search/node_index.rs b/raphtory/src/search/node_index.rs index 27414462b9..39094dacfb 100644 --- a/raphtory/src/search/node_index.rs +++ b/raphtory/src/search/node_index.rs @@ -6,10 +6,8 @@ use crate::{ }, db::{ api::{ - properties::internal::{ - ConstPropertiesOps, TemporalPropertiesOps, TemporalPropertiesRowView, - }, - storage::graph::storage_ops::GraphStorage, + properties::internal::TemporalPropertiesRowView, + storage::graph::storage_ops::GraphStorage, view::IndexSpec, }, graph::node::NodeView, }, @@ -17,14 +15,15 @@ use crate::{ search::{ entity_index::EntityIndex, fields::{NODE_ID, NODE_NAME, NODE_NAME_TOKENIZED, NODE_TYPE, NODE_TYPE_TOKENIZED}, - TOKENIZER, + resolve_props, TOKENIZER, }, }; use raphtory_api::core::storage::{arc_str::ArcStr, dict_mapper::MaybeNew}; use rayon::{prelude::ParallelIterator, slice::ParallelSlice}; use std::{ + collections::HashSet, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; use tantivy::{ collector::TopDocs, @@ -128,6 +127,16 @@ impl NodeIndex { }) } + pub(crate) fn resolve_const_props(&self) -> HashSet { + let props = self.entity_index.const_property_indexes.read(); + resolve_props(&props) + } + + pub(crate) fn resolve_temp_props(&self) -> HashSet { + let props = self.entity_index.temporal_property_indexes.read(); + resolve_props(&props) + } + pub(crate) fn print(&self) -> Result<(), GraphError> { let searcher = self.entity_index.reader.searcher(); let top_docs = searcher.search(&AllQuery, &TopDocs::with_limit(1000))?; @@ -205,17 +214,13 @@ impl NodeIndex { fn index_node_c( &self, node_id: VID, - const_writers: &mut [Option], - const_props: &[(usize, Prop)], + writers: &mut [Option], + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let node_id = node_id.as_u64(); - self.entity_index.index_node_const_properties( - node_id, - const_writers, - const_props.iter().map(|(id, prop)| (*id, prop)), - )?; - - self.entity_index.commit_writers(const_writers) + self.entity_index + .index_node_const_properties(node_id, writers, props)?; + self.entity_index.commit_writers(writers) } fn index_node_t( @@ -226,15 +231,11 @@ impl NodeIndex { node_type: Option, writer: &mut IndexWriter, temporal_writers: &mut [Option], - temporal_props: &[(usize, Prop)], + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let vid_u64 = node_id.inner().as_u64(); - self.entity_index.index_node_temporal_properties( - time, - vid_u64, - temporal_writers, - temporal_props.iter().map(|(id, prop)| (*id, prop)), - )?; + self.entity_index + .index_node_temporal_properties(time, vid_u64, temporal_writers, props)?; // Check if the node document is already in the index, // if it does skip adding a new doc for same node @@ -266,7 +267,11 @@ impl NodeIndex { self.entity_index.index_node_const_properties( node_id, const_writers, - node.properties().constant().iter_id(), + &*node + .properties() + .constant() + .iter_id() + .collect::>(), )?; for (t, temporal_properties) in node.rows() { @@ -274,7 +279,7 @@ impl NodeIndex { t, node_id, temporal_writers, - temporal_properties, + &*temporal_properties, )?; } @@ -285,66 +290,51 @@ impl NodeIndex { } pub(crate) fn index_nodes( + &self, graph: &GraphStorage, - path: Option<&Path>, + path: Option, + index_spec: &IndexSpec, ) -> Result { - let node_index_path = path.as_deref().map(|p| p.join("nodes")); - let node_index = NodeIndex::new(&node_index_path)?; - // Initialize property indexes and get their writers - let const_property_keys = graph.node_meta().const_prop_meta().get_keys().into_iter(); - let const_properties_index_path = node_index_path - .as_deref() - .map(|p| p.join("const_properties")); - let mut const_writers = node_index - .entity_index - .initialize_node_const_property_indexes( - graph, - const_property_keys, - &const_properties_index_path, - )?; + let const_properties_index_path = path.as_deref().map(|p| p.join("const_properties")); + let mut const_writers = self.entity_index.initialize_node_const_property_indexes( + graph.node_meta().const_prop_meta(), + &const_properties_index_path, + &index_spec.node_const_props, + )?; - let temporal_property_keys = graph - .node_meta() - .temporal_prop_meta() - .get_keys() - .into_iter(); - let temporal_properties_index_path = node_index_path - .as_deref() - .map(|p| p.join("temporal_properties")); - let mut temporal_writers = node_index + let temporal_properties_index_path = path.as_deref().map(|p| p.join("temporal_properties")); + let mut temporal_writers = self .entity_index .initialize_node_temporal_property_indexes( - graph, - temporal_property_keys, + graph.node_meta().temporal_prop_meta(), &temporal_properties_index_path, + &index_spec.node_temp_props, )?; // Index nodes in parallel - let mut writer = node_index.entity_index.index.writer(100_000_000)?; + let mut writer = self.entity_index.index.writer(100_000_000)?; let v_ids = (0..graph.count_nodes()).collect::>(); v_ids.par_chunks(128).try_for_each(|v_ids| { for v_id in v_ids { if let Some(node) = graph.node(NodeRef::new((*v_id).into())) { - node_index.index_node(node, &writer, &const_writers, &temporal_writers)?; + self.index_node(node, &writer, &const_writers, &temporal_writers)?; } } Ok::<(), GraphError>(()) })?; // Commit writers - node_index.entity_index.commit_writers(&mut const_writers)?; - node_index - .entity_index - .commit_writers(&mut temporal_writers)?; + self.entity_index.commit_writers(&mut const_writers)?; + self.entity_index.commit_writers(&mut temporal_writers)?; writer.commit()?; // Reload readers - node_index.entity_index.reload_const_property_indexes()?; - node_index.entity_index.reload_temporal_property_indexes()?; - node_index.entity_index.reader.reload()?; + self.entity_index.reload_const_property_indexes()?; + self.entity_index.reload_temporal_property_indexes()?; + self.entity_index.reader.reload()?; - Ok(node_index) + Ok(self.clone()) } pub(crate) fn add_node_update( @@ -359,10 +349,7 @@ impl NodeIndex { .expect("Node for internal id should exist.") .at(t.t()); - let temporal_property_ids = node.temporal_prop_ids(); - let mut temporal_writers = self - .entity_index - .get_temporal_property_writers(temporal_property_ids)?; + let mut temporal_writers = self.entity_index.get_temporal_property_writers(props)?; let mut writer = self.entity_index.index.writer(100_000_000)?; self.index_node_t( @@ -375,6 +362,7 @@ impl NodeIndex { props, )?; + self.entity_index.reload_temporal_property_indexes()?; self.entity_index.reader.reload()?; Ok(()) @@ -382,18 +370,10 @@ impl NodeIndex { pub(crate) fn add_node_constant_properties( &self, - graph: &GraphStorage, node_id: VID, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let node = graph - .node(VID(node_id.as_u64() as usize)) - .expect("Node for internal id should exist."); - - let const_property_ids = node.const_prop_ids(); - let mut const_writers = self - .entity_index - .get_const_property_writers(const_property_ids)?; + let mut const_writers = self.entity_index.get_const_property_writers(props)?; self.index_node_c(node_id, &mut const_writers, props)?; @@ -404,24 +384,16 @@ impl NodeIndex { pub(crate) fn update_node_constant_properties( &self, - graph: &GraphStorage, node_id: VID, props: &[(usize, Prop)], ) -> Result<(), GraphError> { - let node = graph - .node(VID(node_id.as_u64() as usize)) - .expect("Node for internal id should exist."); - - let const_property_ids = node.const_prop_ids(); - let mut const_writers = self - .entity_index - .get_const_property_writers(const_property_ids)?; + let mut const_writers = self.entity_index.get_const_property_writers(props)?; // Delete existing constant property document self.entity_index.delete_const_properties_index_docs( node_id.as_u64(), &mut const_writers, - props.iter().map(|(id, prop)| (*id, prop)), + props, )?; // Reindex the node's constant properties diff --git a/raphtory/src/search/property_index.rs b/raphtory/src/search/property_index.rs index 0372adea8c..008e4150bc 100644 --- a/raphtory/src/search/property_index.rs +++ b/raphtory/src/search/property_index.rs @@ -137,8 +137,17 @@ impl PropertyIndex { let entry = entry?; let path = entry.path(); if path.is_dir() { - let prop_index = Self::load_from_path(&path, is_edge)?; - result.push(Some(prop_index)); + if let Some(file_name) = path.file_name().and_then(|s| s.to_str()) { + if let Ok(prop_id) = file_name.parse::() { + let prop_index = Self::load_from_path(&path, is_edge)?; + + if result.len() <= prop_id { + result.resize(prop_id + 1, None); + } + + result[prop_id] = Some(prop_index); + } + } } } diff --git a/raphtory/src/search/searcher.rs b/raphtory/src/search/searcher.rs index 3e3526088a..7c71ce6641 100644 --- a/raphtory/src/search/searcher.rs +++ b/raphtory/src/search/searcher.rs @@ -81,7 +81,7 @@ mod search_tests { AsNodeFilter, NodeFilter, NodeFilterBuilderOps, PropertyFilterOps, }, }, - prelude::{AdditionOps, Graph, NodeViewOps, PropertyFilter}, + prelude::{AdditionOps, Graph, IndexMutationOps, NodeViewOps, PropertyFilter}, }; fn fuzzy_search_nodes(filter: impl AsNodeFilter) -> Vec { @@ -166,7 +166,9 @@ mod search_tests { AsEdgeFilter, EdgeFilter, EdgeFilterOps, PropertyFilterOps, }, }, - prelude::{AdditionOps, EdgeViewOps, Graph, NodeViewOps, PropertyFilter}, + prelude::{ + AdditionOps, EdgeViewOps, Graph, IndexMutationOps, NodeViewOps, PropertyFilter, + }, }; fn fuzzy_search_edges(filter: impl AsEdgeFilter) -> Vec<(String, String)> { diff --git a/raphtory/src/serialise/incremental.rs b/raphtory/src/serialise/incremental.rs index 945f2c6f9b..47266bfe7f 100644 --- a/raphtory/src/serialise/incremental.rs +++ b/raphtory/src/serialise/incremental.rs @@ -1,6 +1,6 @@ use super::GraphFolder; #[cfg(feature = "search")] -use crate::prelude::SearchableGraphOps; +use crate::prelude::IndexMutationOps; use crate::{ core::{ utils::errors::{GraphError, WriteError}, @@ -10,7 +10,7 @@ use crate::{ api::{storage::storage::Storage, view::MaterializedGraph}, graph::views::deletion_graph::PersistentGraph, }, - prelude::{Graph, StableDecode}, + prelude::{AdditionOps, Graph, StableDecode}, serialise::{ serialise::{CacheOps, InternalStableDecode, StableEncode}, ProtoGraph, @@ -294,7 +294,7 @@ impl InternalCache for MaterializedGraph { } } -impl CacheOps for G { +impl CacheOps for G { fn cache(&self, path: impl Into) -> Result<(), GraphError> { let folder = path.into(); self.encode(&folder)?; diff --git a/raphtory/src/serialise/mod.rs b/raphtory/src/serialise/mod.rs index 68bd4a6f68..8b007c7a04 100644 --- a/raphtory/src/serialise/mod.rs +++ b/raphtory/src/serialise/mod.rs @@ -12,7 +12,7 @@ mod proto { } #[cfg(feature = "search")] -use crate::prelude::SearchableGraphOps; +use crate::prelude::IndexMutationOps; use crate::{ core::utils::errors::GraphError, db::api::view::MaterializedGraph, prelude::GraphViewOps, serialise::metadata::GraphMetadata, @@ -31,7 +31,7 @@ const META_FILE_NAME: &str = ".raph"; #[derive(Clone, Debug)] pub struct GraphFolder { - root_folder: PathBuf, + pub root_folder: PathBuf, prefer_zip_format: bool, } diff --git a/raphtory/src/serialise/serialise.rs b/raphtory/src/serialise/serialise.rs index 5b9b9e08b8..9c017f7f7b 100644 --- a/raphtory/src/serialise/serialise.rs +++ b/raphtory/src/serialise/serialise.rs @@ -1,6 +1,6 @@ use super::{proto_ext::PropTypeExt, GraphFolder}; #[cfg(feature = "search")] -use crate::prelude::SearchableGraphOps; +use crate::prelude::IndexMutationOps; use crate::{ core::{ entities::{graph::tgraph::TemporalGraph, LayerIds}, @@ -18,7 +18,7 @@ use crate::{ }, graph::views::deletion_graph::PersistentGraph, }, - prelude::Graph, + prelude::{AdditionOps, Graph}, serialise::{ proto::{self, graph_update::*, new_meta::*, new_node::Gid}, proto_ext, @@ -43,7 +43,7 @@ macro_rules! zip_tprop_updates { }; } -pub trait StableEncode: StaticGraphViewOps { +pub trait StableEncode: StaticGraphViewOps + AdditionOps { fn encode_to_proto(&self) -> proto::Graph; fn encode_to_vec(&self) -> Vec { self.encode_to_proto().encode_to_vec() @@ -55,7 +55,7 @@ pub trait StableEncode: StaticGraphViewOps { } } -pub trait StableDecode: InternalStableDecode + StaticGraphViewOps { +pub trait StableDecode: InternalStableDecode + StaticGraphViewOps + AdditionOps { fn decode(path: impl Into) -> Result { let folder = path.into(); let graph = Self::decode_from_path(&folder)?; @@ -67,7 +67,7 @@ pub trait StableDecode: InternalStableDecode + StaticGraphViewOps { } } -impl StableDecode for T {} +impl StableDecode for T {} pub trait InternalStableDecode: Sized { fn decode_from_proto(graph: &proto::Graph) -> Result; @@ -1475,250 +1475,4 @@ mod proto_test { Prop::from_arr::(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), )); } - - #[cfg(feature = "search")] - mod test_index_io { - use crate::{ - core::{utils::errors::GraphError, Prop}, - db::{ - api::{ - mutation::internal::{InternalAdditionOps, InternalPropertyAdditionOps}, - view::{internal::InternalStorageOps, StaticGraphViewOps}, - }, - graph::views::filter::model::{AsNodeFilter, NodeFilter, NodeFilterBuilderOps}, - }, - prelude::{ - AdditionOps, CacheOps, Graph, GraphViewOps, NodeViewOps, PropertyAdditionOps, - SearchableGraphOps, StableDecode, StableEncode, - }, - serialise::GraphFolder, - }; - use raphtory_api::core::{storage::arc_str::ArcStr, utils::logging::global_info_logger}; - - fn init_graph(graph: G) -> G - where - G: StaticGraphViewOps - + AdditionOps - + InternalAdditionOps - + InternalPropertyAdditionOps - + PropertyAdditionOps, - { - graph - .add_node( - 1, - "Alice", - vec![("p1", Prop::U64(2u64))], - Some("fire_nation"), - ) - .unwrap(); - graph - } - - fn assert_search_results( - graph: &Graph, - filter: &T, - expected: Vec<&str>, - ) { - let res = graph - .search_nodes(filter.clone(), 2, 0) - .unwrap() - .into_iter() - .map(|n| n.name()) - .collect::>(); - assert_eq!(res, expected); - } - - #[test] - fn test_create_no_index_persist_no_index_on_encode_load_no_index_on_decode() { - // No index persisted since it was never created - let graph = init_graph(Graph::new()); - - let err = graph - .search_nodes(NodeFilter::name().eq("Alice"), 2, 0) - .expect_err("Expected error since index was not created"); - assert!(matches!(err, GraphError::IndexNotCreated)); - - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let index = graph.get_storage().unwrap().index.get(); - assert!(index.is_none()); - } - - #[test] - fn test_create_index_persist_index_on_encode_load_index_on_decode() { - let graph = init_graph(Graph::new()); - - // Created index - graph.create_index().unwrap(); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - - // Persisted both graph and index - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Loaded index that was persisted - let graph = Graph::decode(path).unwrap(); - let index = graph.get_storage().unwrap().index.get(); - assert!(index.is_some()); - - assert_search_results(&graph, &filter, vec!["Alice"]); - } - - #[test] - fn test_create_index_persist_index_on_encode_update_index_load_persisted_index_on_decode() { - let graph = init_graph(Graph::new()); - - // Created index - graph.create_index().unwrap(); - - let filter1 = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter1, vec!["Alice"]); - - // Persisted both graph and index - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Updated both graph and index - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - ) - .unwrap(); - let filter2 = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - - // Loaded index that was persisted - let graph = Graph::decode(path).unwrap(); - let index = graph.get_storage().unwrap().index.get(); - assert!(index.is_some()); - assert_search_results(&graph, &filter1, vec!["Alice"]); - assert_search_results(&graph, &filter2, Vec::<&str>::new()); - - // Updating and encode the graph and index should decode the updated the graph as well as index - // So far we have the index that was created and persisted for the first time - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - ) - .unwrap(); - let filter2 = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - - // Should persist the updated graph and index - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Should load the updated graph and index - let graph = Graph::decode(path).unwrap(); - let index = graph.get_storage().unwrap().index.get(); - assert!(index.is_some()); - assert_search_results(&graph, &filter1, vec!["Alice"]); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - } - - #[test] - fn test_zip_encode_decode_index() { - let graph = init_graph(Graph::new()); - graph.create_index().unwrap(); - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - let folder = GraphFolder::new_as_zip(path); - graph.encode(folder.root_folder).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let node = graph.node("Alice").unwrap(); - let node_type = node.node_type(); - assert_eq!(node_type, Some(ArcStr::from("fire_nation"))); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - } - - #[test] - fn test_create_index_in_ram() { - global_info_logger(); - - let graph = init_graph(Graph::new()); - graph.create_index_in_ram().unwrap(); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let index = graph.get_storage().unwrap().index.get(); - assert!(index.is_none()); - - let results = graph.search_nodes(filter.clone(), 2, 0); - assert!(matches!(results, Err(GraphError::IndexNotCreated))); - } - - #[test] - fn test_cached_graph_view() { - global_info_logger(); - let graph = init_graph(Graph::new()); - graph.create_index().unwrap(); - - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.cache(path).unwrap(); - - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - ) - .unwrap(); - graph.write_updates().unwrap(); - - let graph = Graph::decode(path).unwrap(); - let filter = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter, vec!["Tommy"]); - } - - #[test] - fn test_cached_graph_view_create_index_after_graph_is_cached() { - global_info_logger(); - let graph = init_graph(Graph::new()); - - let binding = tempfile::TempDir::new().unwrap(); - let path = binding.path(); - graph.cache(path).unwrap(); - // Creates index in a temp dir within graph dir - graph.create_index().unwrap(); - - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - ) - .unwrap(); - graph.write_updates().unwrap(); - - let graph = Graph::decode(path).unwrap(); - let filter = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter, vec!["Tommy"]); - } - } }