Show all dependent items on stacked tables (#3251)

Author: soininen

spinetoolbox/relationship_graphs.py

@@ -0,0 +1,91 @@
+######################################################################################################################
+# Copyright (C) 2017-2022 Spine project consortium
+# Copyright Spine Toolbox contributors
+# This file is part of Spine Toolbox.
+# Spine Toolbox is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General
+# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option)
+# any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+######################################################################################################################
+from __future__ import annotations
+import networkx as nx
+from PySide6.QtCore import QObject, Slot
+from spinedb_api import DatabaseMapping
+from spinedb_api.helpers import ItemType
+from spinedb_api.temp_id import TempId
+from .fetch_parent import DBMapMixedItems
+
+
+class GraphBase(QObject):
+    def __init__(self, parent: QObject | None):
+        super().__init__(parent)
+        self._graphs: dict[DatabaseMapping, nx.DiGraph] = {}
+
+    def is_any_id_reachable(self, db_map: DatabaseMapping, source_id: TempId, target_ids: set[TempId]) -> bool:
+        if db_map not in self._graphs:
+            self._graphs[db_map] = self._build_graph(db_map)
+        graph = self._graphs[db_map]
+        relationship_ids = list(graph.predecessors(source_id))
+        while relationship_ids:
+            relationship_id = relationship_ids.pop(-1)
+            if relationship_id in target_ids:
+                return True
+            relationship_ids += graph.predecessors(relationship_id)
+        return False
+
+    @staticmethod
+    def _build_graph(db_map: DatabaseMapping) -> nx.DiGraph:
+        raise NotImplementedError()
+
+    @Slot(object)
+    def invalidate_caches(self, db_map: DatabaseMapping) -> None:
+        if db_map in self._graphs:
+            del self._graphs[db_map]
+
+    @Slot(str, object)
+    def maybe_invalidate_caches_after_data_changed(self, item_type: ItemType, db_map_data: DBMapMixedItems) -> None:
+        if item_type != "entity_class" and item_type != "superclass_subclass":
+            return
+        for db_map in db_map_data:
+            if db_map in self._graphs:
+                del self._graphs[db_map]
+
+    @Slot(str, object)
+    def maybe_invalidate_caches_after_fetch(self, item_type: ItemType, db_map: DatabaseMapping) -> None:
+        if (item_type != "entity_class" and item_type != "superclass_subclass") or db_map not in self._graphs:
+            return
+        del self._graphs[db_map]
+
+
+class RelationshipClassGraph(GraphBase):
+    @staticmethod
+    def _build_graph(db_map: DatabaseMapping) -> nx.DiGraph:
+        graph = _build_graph(db_map, "entity_class", "dimension_id_list")
+        for superclass_subclass in db_map.mapped_table("superclass_subclass").values():
+            if not superclass_subclass.is_valid():
+                continue
+            graph.add_edge(superclass_subclass["subclass_id"], superclass_subclass["superclass_id"])
+        return graph
+
+
+class RelationshipGraph(GraphBase):
+    @staticmethod
+    def _build_graph(db_map: DatabaseMapping) -> nx.DiGraph:
+        return _build_graph(db_map, "entity", "element_id_list")
+
+
+def _build_graph(db_map: DatabaseMapping, item_type: ItemType, id_list_name: str) -> nx.DiGraph:
+    graph = nx.DiGraph()
+    for item in db_map.mapped_table(item_type).values():
+        if not item.is_valid():
+            continue
+        item_id = item["id"]
+        if not graph.has_node(item_id):
+            graph.add_node(item_id)
+        for dimension_id in item[id_list_name]:
+            if not graph.has_node(dimension_id):
+                graph.add_node(dimension_id)
+            graph.add_edge(dimension_id, item_id)
+    return graph

spinetoolbox/spine_db_editor/mvcmodels/compound_models.py

@@ -59,7 +59,7 @@ class CompoundStackedModel(CompoundTableModel):
     _ENTITY_CLASS_ID_FIELD: ClassVar[str] = "entity_class_id"
     FIELDS_REQUIRING_FILTER_DATA_CONVERSION: ClassVar[set[str]] = set()
 
-    def __init__(self, parent: QObject, db_mngr: SpineDBManager, *db_maps):
+    def __init__(self, parent: QObject, db_mngr: SpineDBManager, *db_maps: DatabaseMapping):
         """
         Args:
             parent: the parent object
@@ -218,7 +218,9 @@ def filter_accepts_model(self, model: SingleModelBase) -> bool:
         if model.db_map not in self._filter_class_ids:
             return False
         class_ids = self._filter_class_ids[model.db_map]
-        return model.entity_class_id in class_ids or not class_ids.isdisjoint(model.dimension_id_list)
+        if model.entity_class_id in class_ids:
+            return True
+        return self.db_mngr.relationship_class_graph.is_any_id_reachable(model.db_map, model.entity_class_id, class_ids)
 
     def stop_invalidating_filter(self) -> None:
         """Stops invalidating the filter."""

spinetoolbox/spine_db_editor/mvcmodels/single_models.py

@@ -87,19 +87,17 @@ def __init__(
         self.committed = committed
 
     def __lt__(self, other):
-        if self.entity_class_name == other.entity_class_name:
+        entity_class = self.db_map.mapped_table("entity_class")[self.entity_class_id]
+        class_name = entity_class["name"]
+        other_entity_class = other.db_map.mapped_table("entity_class")[other.entity_class_id]
+        other_class_name = other_entity_class["name"]
+        if class_name == other_class_name:
             return self.db_mngr.name_registry.display_name(
                 self.db_map.sa_url
             ) < self.db_mngr.name_registry.display_name(other.db_map.sa_url)
-        keys = {}
-        for side, model in {"left": self, "right": other}.items():
-            dim = len(model.dimension_id_list)
-            class_name = model.entity_class_name
-            keys[side] = (
-                dim,
-                class_name,
-            )
-        return keys["left"] < keys["right"]
+        keys = (len(entity_class["dimension_id_list"]), class_name)
+        other_keys = (len(other_entity_class["dimension_id_list"]), other_class_name)
+        return keys < other_keys
 
     @property
     def item_type(self) -> str:
@@ -126,24 +124,14 @@ def _convert_to_db(self, item: dict) -> dict:
     def _references(self) -> dict[str, tuple[str, str | None]]:
         raise NotImplementedError()
 
-    @property
-    def entity_class_name(self) -> str:
-        entity_class_table = self.db_map.mapped_table("entity_class")
-        return entity_class_table[self.entity_class_id]["name"]
-
-    @property
-    def dimension_id_list(self) -> list[TempId]:
-        entity_class_table = self.db_map.mapped_table("entity_class")
-        return entity_class_table[self.entity_class_id]["dimension_id_list"]
-
     def item_id(self, row: int) -> TempId:
-        """Returns parameter id for row.
+        """Returns item's id for row.
 
         Args:
             row: row index
 
         Returns:
-            parameter id
+            item's id
         """
         return self._main_data[row]
 
@@ -269,30 +257,24 @@ def set_filter_entity_ids(self, entity_selection: EntitySelection) -> bool:
                     entity_ids = selected_entities_by_class[self.entity_class_id]
                 else:
                     entity_ids = set()
-                    dimension_id_list = self.db_map.mapped_table("entity_class")[self.entity_class_id][
-                        "dimension_id_list"
-                    ]
-                    for dimension_id in dimension_id_list:
-                        if dimension_id not in selected_entities_by_class:
-                            continue
-                        selected_entities = selected_entities_by_class[dimension_id]
-                        if selected_entities is Asterisk:
-                            continue
-                        entity_ids |= selected_entities
-                    if dimension_id_list and not entity_ids:
-                        entity_ids = Asterisk
+                    for class_id, entity_selection in selected_entities_by_class.items():
+                        if entity_selection is Asterisk:
+                            entity_ids = Asterisk
+                            break
+                        entity_ids.update(entity_selection)
         if entity_ids == self._filter_entity_ids:
             return False
         self._filter_entity_ids = entity_ids
         return True
 
     def filter_accepts_item(self, item: PublicItem) -> bool:
-        """Reimplemented to also account for the entity and alternative filter."""
+        """Reimplemented to also account for the entity filter."""
         if self._filter_entity_ids is Asterisk:
             return super().filter_accepts_item(item)
-        entity_accepts = item[
-            self._ENTITY_ID_FIELD
-        ] in self._filter_entity_ids or not self._filter_entity_ids.isdisjoint(item["element_id_list"])
+        entity_id = item[self._ENTITY_ID_FIELD]
+        entity_accepts = entity_id in self._filter_entity_ids or self.db_mngr.relationship_graph.is_any_id_reachable(
+            self.db_map, entity_id, self._filter_entity_ids
+        )
         return entity_accepts and super().filter_accepts_item(item)
 
 
@@ -314,7 +296,7 @@ def set_filter_alternative_ids(self, alternative_selection: AlternativeSelection
         return True
 
     def filter_accepts_item(self, item: PublicItem) -> bool:
-        """Reimplemented to also account for the entity and alternative filter."""
+        """Reimplemented to also account for the alternative filter."""
         if self._filter_alternative_ids is Asterisk:
             return super().filter_accepts_item(item)
         return item["alternative_id"] in self._filter_alternative_ids and super().filter_accepts_item(item)
@@ -445,7 +427,7 @@ class EntityMixin:
 
     def update_items_in_db(self, items: list[dict]) -> None:
         """Overridden to create entities on the fly first."""
-        class_name = self.entity_class_name
+        class_name = self.db_map.mapped_table("entity_class")[self.entity_class_id]["name"]
         for item in items:
             item["entity_class_name"] = class_name
         entities = []

spinetoolbox/spine_db_editor/selection_for_filtering.py

@@ -39,7 +39,7 @@ def _update_class_or_entity_selection(self, selected: QItemSelection, deselected
         class_ids = {}
         entity_ids = {}
         selection = self._selection_model.selection().indexes()
-        for index in _include_parents(selection):
+        for index in selection:
             if index.column() != 0:
                 continue
             if not index.parent().isValid():
@@ -107,18 +107,6 @@ def _remove_surplus_entity_id_asterisks(entity_selection: EntitySelection) -> No
                 class_selection[class_id] = set()
 
 
-def _include_parents(indexes: Iterable[QModelIndex]) -> Iterator[QModelIndex]:
-    parents = {}
-    for index in indexes:
-        yield index
-        parent = index.parent()
-        if not parent.isValid() or parent.data() == "root":
-            continue
-        parents[(parent.row(), parent.column(), id(parent.internalPointer()))] = parent
-    if parents:
-        yield from _include_parents(parents.values())
-
-
 class AlternativeSelectionForFiltering(QObject):
     alternative_selection_changed = Signal(object)
 

spinetoolbox/spine_db_manager.py

@@ -65,6 +65,7 @@
 from .helpers import DBMapDictItems, DBMapPublicItems, busy_effect, normcase_database_url_path, plain_to_tool_tip
 from .mvcmodels.shared import INVALID_TYPE, PARAMETER_TYPE_VALIDATION_ROLE, PARSED_ROLE, TYPE_NOT_VALIDATED, VALID_TYPE
 from .parameter_type_validation import ParameterTypeValidator
+from .relationship_graphs import RelationshipClassGraph, RelationshipGraph
 from .spine_db_commands import (
     AddItemsCommand,
     AddUpdateItemsCommand,
@@ -94,21 +95,21 @@ class SpineDBManager(QObject):
     """Emitted whenever items are added to a DB.
 
     Args:
-        str: item type, such as "object_class"
+        str: item type, such as "entity_class"
         object: a dictionary mapping DatabaseMapping to list of added dict-items.
     """
     items_updated = Signal(str, object)
     """Emitted whenever items are updated in a DB.
 
     Args:
-        str: item type, such as "object_class"
+        str: item type, such as "entity_class"
         object: a dictionary mapping DatabaseMapping to list of updated dict-items.
     """
     items_removed = Signal(str, object)
     """Emitted whenever items are removed from a DB.
 
     Args:
-        str: item type, such as "object_class"
+        str: item type, such as "entity_class"
         object: a dictionary mapping DatabaseMapping to list of updated dict-items.
     """
     database_clean_changed = Signal(object, bool)
@@ -118,6 +119,19 @@ class SpineDBManager(QObject):
         object: database mapping
         bool: True if database has become clean, False if it became dirty
     """
+    more_data_fetched = Signal(object, str)
+    """Emitted whenever data is fetched from a database.
+
+    Args:
+        object: database mapping
+        str: item type, such as "entity_class"
+    """
+    database_refreshed = Signal(object)
+    """Emitted whenever database is refreshed.
+
+    Args:
+        object: database mapping
+    """
     database_reset = Signal(object)
     """Emitted whenever database is reset.
 
@@ -140,6 +154,14 @@ def __init__(self, settings: QSettings, parent: Optional[QObject], synchronous:
         self._lock_lock = RLock()
         self._db_locks: dict[DatabaseMapping, RLock] = {}
         self.listeners: dict[DatabaseMapping, set[object]] = {}
+        self.relationship_class_graph = RelationshipClassGraph(self)
+        self.relationship_graph = RelationshipGraph(self)
+        for graph in (self.relationship_class_graph, self.relationship_class_graph):
+            for signal in (self.items_added, self.items_updated, self.items_removed):
+                signal.connect(graph.maybe_invalidate_caches_after_data_changed)
+            for signal in (self.database_refreshed, self.database_reset):
+                signal.connect(graph.invalidate_caches)
+            self.more_data_fetched.connect(graph.maybe_invalidate_caches_after_fetch)
         self.undo_stack: dict[DatabaseMapping, AgedUndoStack] = {}
         self.undo_action: dict[DatabaseMapping, QAction] = {}
         self.redo_action: dict[DatabaseMapping, QAction] = {}
@@ -343,6 +365,8 @@ def close_session(self, url: str) -> None:
                 del self._db_locks[db_map]
         del self._validated_values["parameter_definition"][id(db_map)]
         del self._validated_values["parameter_value"][id(db_map)]
+        self.relationship_class_graph.invalidate_caches(db_map)
+        self.relationship_graph.invalidate_caches(db_map)
         self.undo_stack[db_map].cleanChanged.disconnect()
         del self.undo_stack[db_map]
         del self.undo_action[db_map]

spinetoolbox/spine_db_worker.py

@@ -183,6 +183,7 @@ def _busy_db_map_fetch_more(self, item_type):
     def _handle_query_advanced(self, item_type, chunk):
         self._populate_commit_cache(item_type, chunk)
         self._db_mngr.update_icons(self._db_map, item_type, chunk)
+        self._db_mngr.more_data_fetched.emit(self._db_map, item_type)
         parents = self._parents_fetching.pop(item_type, ())
         if parents and not self._db_map.closed:
             self._query_advanced.emit(parents)

tests/spine_db_editor/mvcmodels/test_compound_models.py

@@ -75,7 +75,8 @@ def test_model_updates_when_entity_class_is_removed(self):
         fetch_model(model)
         self.assertEqual(model.rowCount(), 3)
         model.set_entity_selection_for_filtering({self._db_map: {entity_class_2["id"]: Asterisk}})
-        model.refresh()
+        while model.rowCount() == 3:
+            QApplication.processEvents()
         self.assertEqual(model.rowCount(), 2)
         self._db_mngr.remove_items({self._db_map: {"entity_class": [entity_class_2["id"]]}})
         while model.rowCount() == 2:

tests/spine_db_editor/test_selection_for_filtering.py

@@ -191,9 +191,7 @@ def test_select_entity_with_multiple_db_maps(self, db_editor, tmp_path, logger):
             assert watch_database_index.data() == "db1, db2"
             selection = QItemSelection(iron_index, iron_database_index)
             selection_model.select(selection, QItemSelectionModel.SelectionFlag.Select)
-            mock_signal.emit.assert_called_once_with(
-                {db_map1: {entity_class1["id"]: {entity1b["id"]}}, db_map2: {entity_class2["id"]: set()}}
-            )
+            mock_signal.emit.assert_called_once_with({db_map1: {entity_class1["id"]: {entity1b["id"]}}})
             mock_signal.emit.reset_mock()
             selection = QItemSelection(watch_index, watch_database_index)
             selection_model.select(selection, QItemSelectionModel.SelectionFlag.Select)

tests/spine_db_editor/widgets/test_SpineDBEditorFilter.py

@@ -48,11 +48,13 @@ def _assert_filter(self, filtered_values):
             data = self._parameter_data(model, *fields)
             values = filtered_values[model]
             unfiltered_count = len(data)
-            self.assertTrue(all(value in data for value in values))
+            for value in values:
+                self.assertIn(value, data)
             model.refresh()
             data = self._parameter_data(model, *fields)
             filtered_count = len(data)
-            self.assertTrue(all(value not in data for value in values))
+            for value in values:
+                self.assertNotIn(value, data)
             # Check that only the items that were supposed to be filtered were actually filtered.
             self.assertEqual(filtered_count, unfiltered_count - len(values))
 
@@ -119,8 +121,6 @@ def test_filter_parameter_tables_per_entity_class_and_entity_cross_selection(sel
             self.spine_db_editor.parameter_definition_model: [],
             self.spine_db_editor.parameter_value_model: [
                 ("dog", ("pluto",)),
-                ("fish__dog", ("nemo", "pluto")),
-                ("dog__fish", ("pluto", "nemo")),
             ],
         }
         self._assert_filter(filtered_values)