Merge remote-tracking branch 'origin/main' into pgm/remove-id-counter

Signed-off-by: Thijs Baaijen <13253091+Thijss@users.noreply.github.com>

# Conflicts:
#	tests/unit/model/grids/test_base.py

Author: Thijss

.github/workflows/sonar.yml

@@ -46,7 +46,7 @@ jobs:
 
       - name: SonarCloud Scan
         if: ${{ (github.event_name == 'push') || (github.event.pull_request.head.repo.owner.login == 'PowerGridModel') }}
-        uses: SonarSource/sonarqube-scan-action@299e4b793aaa83bf2aba7c9c14bedbb485688ec4 # v7.1.0
+        uses: SonarSource/sonarqube-scan-action@59db25f34e16620e48ab4bb9e4a5dce155cb5432 # v8.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

pyproject.toml

@@ -160,7 +160,7 @@ select = [
   # raise vanilla args
   "TRY002",
   # pytest-style
-  #"PT",
+  "PT",
 ]
 # Allow fix for all enabled rules (when `--fix`) is provided.
 fixable = ["ALL"]

src/power_grid_model_ds/_core/model/arrays/base/array.py

@@ -19,7 +19,11 @@
 from power_grid_model_ds._core.model.arrays.base._string import convert_array_to_string
 from power_grid_model_ds._core.model.arrays.base.errors import ArrayDefinitionError
 from power_grid_model_ds._core.model.constants import EMPTY_ID, empty
-from power_grid_model_ds._core.utils.misc import array_equal_with_nan, get_inherited_attrs
+from power_grid_model_ds._core.utils.misc import (
+    array_equal_with_nan,
+    combine_attribute_from_parent_classes,
+    get_public_annotations,
+)
 
 # pylint: disable=missing-function-docstring, too-many-public-methods
 
@@ -44,9 +48,15 @@ class FancyArray(ABC):  # noqa: B024
         >>>     name: NDArray[np.str_]
         >>>     value: NDArray[np.float64]
 
+    Note on default values:
+        Default values for columns can be defined by adding a class attribute _defaults,
+        which is a dictionary mapping column names to their default values.
+        The _defaults attribute is inherited by child classes.
+
     Note on string-columns:
         The default length for string columns is stored in _DEFAULT_STR_LENGTH.
         To change this, you can set the _str_lengths class attribute.
+        The _str_lengths attribute is inherited by child classes.
 
     Example:
         >>> class MyArray(FancyArray):
@@ -74,13 +84,13 @@ def data(self) -> NDArray:
     @classmethod
     @lru_cache
     def get_defaults(cls) -> dict[str, Any]:
-        return get_inherited_attrs(cls, "_defaults")["_defaults"]
+        return combine_attribute_from_parent_classes(cls, "_defaults", attribute_type=dict)
 
     @classmethod
     @lru_cache
     def get_dtype(cls):
-        annotations = get_inherited_attrs(cls, "_str_lengths")
-        str_lengths = annotations.pop("_str_lengths")
+        annotations = get_public_annotations(cls)
+        str_lengths = combine_attribute_from_parent_classes(cls, "_str_lengths", dict)
         dtypes = {}
         for name, dtype in annotations.items():
             if len(dtype.__args__) > 1:

src/power_grid_model_ds/_core/model/grids/_modify.py

@@ -39,7 +39,7 @@ def add_node(grid: "Grid", node: NodeArray) -> None:
     )
     grid._append(array=node)  # noqa # pylint: disable=protected-access
     grid.graphs.add_node_array(node_array=node)
-    _logger.debug("added node %d", node.id)
+    _logger.debug("added node %s", node.id.tolist())
 
 
 def add_branch(grid: "Grid", branch: BranchArray) -> None:
@@ -52,33 +52,43 @@ def add_branch(grid: "Grid", branch: BranchArray) -> None:
     grid._append(array=branch)  # noqa # pylint: disable=protected-access
     grid.graphs.add_branch_array(branch_array=branch)
 
-    _logger.debug("added branch %d from %d to %d", branch.id, branch.from_node, branch.to_node)
+    _logger.debug(
+        "added branch %s from %s to %s", branch.id.tolist(), branch.from_node.tolist(), branch.to_node.tolist()
+    )
 
 
 def make_active(grid: "Grid", branch: BranchArray) -> None:
     """See Grid.make_active()"""
     array_field = grid.find_array_field(branch.__class__)
     array_attr = getattr(grid, array_field.name)
     branch_mask = array_attr.id == branch.id
+    already_active = bool(array_attr[branch_mask].is_active)
     array_attr.from_status[branch_mask] = 1
     array_attr.to_status[branch_mask] = 1
     setattr(grid, array_field.name, array_attr)
 
-    grid.graphs.make_active(branch=branch)
-    _logger.debug("activated branch %d", branch.id)
+    if not already_active:
+        grid.graphs.make_active(branch=branch)
+    else:
+        _logger.warning("Branch %s is already active", branch.id.tolist())
+    _logger.debug("activated branch %s", branch.id.tolist())
 
 
 def make_inactive(grid, branch: BranchArray, at_to_side: bool = True) -> None:
     """See Grid.make_inactive()"""
     array_field = grid.find_array_field(branch.__class__)
     array_attr = getattr(grid, array_field.name)
     branch_mask = array_attr.id == branch.id
+    already_inactive = bool(~array_attr[branch_mask].is_active)
     status_side = "to_status" if at_to_side else "from_status"
     array_attr[status_side][branch_mask] = 0
     setattr(grid, array_field.name, array_attr)
 
-    grid.graphs.make_inactive(branch=branch)
-    _logger.debug("deactivated branch %d", branch.id)
+    if not already_inactive:
+        grid.graphs.make_inactive(branch=branch)
+    else:
+        _logger.warning("Branch %s is already inactive", branch.id.tolist())
+    _logger.debug("deactivated branch %s", branch.id.tolist())
 
 
 def delete_node(grid: "Grid", node: NodeArray) -> None:
@@ -123,23 +133,25 @@ def delete_node(grid: "Grid", node: NodeArray) -> None:
 
     grid.graphs.delete_node(node=node)
     grid.rebuild_ids()
-    _logger.debug("deleted node %d", node.id)
+    _logger.debug("deleted node %s", node.id.tolist())
 
 
 def delete_branch(grid: "Grid", branch: BranchArray) -> None:
     """See Grid.delete_branch()"""
     _delete_branch_array(branch=branch, grid=grid)
     grid.graphs.delete_branch(branch=branch)
     grid.rebuild_ids()
-    _logger.debug("""deleted branch %d from %d to %d""", branch.id, branch.from_node, branch.to_node)
+    _logger.debug(
+        "deleted branch %s from %s to %s", branch.id.tolist(), branch.from_node.tolist(), branch.to_node.tolist()
+    )
 
 
 def delete_branch3(grid: "Grid", branch: Branch3Array) -> None:
     """See Grid.delete_branch3()"""
     _delete_branch_array(branch=branch, grid=grid)
     grid.graphs.delete_branch3(branch=branch)
     grid.rebuild_ids()
-    _logger.debug("deleted branch3 %d", branch.id)
+    _logger.debug("deleted branch3 %s", branch.id.tolist())
 
 
 def _delete_branch_array(branch: BranchArray | Branch3Array, grid: "Grid"):
@@ -166,4 +178,4 @@ def delete_appliance(grid: "Grid", appliance: ApplianceArray) -> None:
     grid.asym_power_sensor = grid.asym_power_sensor.exclude(measured_object=appliance.id)
     grid.voltage_regulator = grid.voltage_regulator.exclude(regulated_object=appliance.id)
     grid.rebuild_ids()
-    _logger.debug("deleted appliance %d", appliance.id)
+    _logger.debug("deleted appliance %s", appliance.id.tolist())

src/power_grid_model_ds/_core/model/grids/_search.py

@@ -22,11 +22,12 @@
 def get_branches(grid: "Grid") -> BranchArray:
     """see Grid.get_branches()"""
     branch_dtype = BranchArray.get_dtype()
-    branches = BranchArray()
-    for array in grid.branch_arrays:
-        new_branch = BranchArray(data=array.data[list(branch_dtype.names)])
-        branches = fp.concatenate(branches, new_branch)
-    return branches
+    branch_columns = list(branch_dtype.names)
+    branch_arrays = get_branch_arrays(grid)
+    consistent_branch_arrays = [array[branch_columns] for array in branch_arrays if array.size]
+    if len(consistent_branch_arrays) == 0:
+        return BranchArray()
+    return fp.concatenate(BranchArray(), *consistent_branch_arrays)
 
 
 def get_branch_arrays(grid: "Grid") -> list[BranchArray]:

src/power_grid_model_ds/_core/model/grids/base.py

@@ -46,7 +46,12 @@
     get_nearest_substation_node,
     get_typed_branches,
 )
-from power_grid_model_ds._core.model.grids.serialization.json import deserialize_from_json, serialize_to_json
+from power_grid_model_ds._core.model.grids.serialization.json import (
+    deserialize_from_json,
+    deserialize_from_json_string,
+    serialize_to_json,
+    serialize_to_json_string,
+)
 from power_grid_model_ds._core.model.grids.serialization.pickle import load_grid_from_pickle, save_grid_to_pickle
 from power_grid_model_ds._core.model.grids.serialization.string import (
     deserialize_from_str,
@@ -464,20 +469,53 @@ def merge(self, other_grid, mode: Literal["keep_ids", "recalculate_ids"]):
         """
         return merge_grids(self, other_grid, mode)
 
-    def serialize(self, path: Path, **kwargs) -> Path:
+    @overload
+    def serialize(self, path: Path, mode: Literal["json"] = "json", **kwargs) -> Path: ...
+
+    @overload
+    def serialize(self, path: None = None, *, mode: Literal["json_string"], **kwargs) -> str: ...
+
+    def serialize(self, path=None, mode: Literal["json", "json_string"] = "json", **kwargs):
         """Serialize the grid.
 
         Args:
-            path: Destination file path to write JSON to.
-            **kwargs: Additional keyword arguments forwarded to ``json.dump``
+            path: Destination file path. Required when mode is ``"json"``, ignored otherwise.
+            mode: Serialization target. Use ``"json"`` (default) to write a JSON file, or ``"json_string"`` to
+                return a JSON string.
+            **kwargs: Additional keyword arguments forwarded to ``json.dump`` / ``json.dumps``.
         Returns:
-            Path: The path where the file was saved.
+            Path when mode is ``"json"``, str when mode is ``"json_string"``.
         """
-        return serialize_to_json(grid=self, path=path, strict=True, **kwargs)
+        match mode:
+            case "json_string":
+                return serialize_to_json_string(grid=self, **kwargs)
+            case "json":
+                if not isinstance(path, Path):
+                    raise TypeError("path must be a Path when mode='json'")
+                return serialize_to_json(grid=self, path=path, strict=True, **kwargs)
+            case _:
+                raise ValueError(f"Invalid mode '{mode}'. Expected 'json' or 'json_string'.")
+
+    @classmethod
+    def from_json_string(cls: type[Self], json_string: str) -> Self:
+        """Deserialize the grid from a JSON string.
+
+        Args:
+            json_string: A JSON string as produced by ``serialize(mode="json_string")``.
+        Returns:
+            Self: The deserialized grid instance.
+        """
+        return deserialize_from_json_string(json_string=json_string, target_grid_class=cls)
 
     @classmethod
     def deserialize(cls: type[Self], path: Path) -> Self:
-        """Deserialize the grid."""
+        """Deserialize the grid from a JSON file.
+
+        Args:
+            path: Path to the JSON file.
+        Returns:
+            Self: The deserialized grid instance.
+        """
         return deserialize_from_json(path=path, target_grid_class=cls)
 
     def rebuild_graphs(self) -> None:

src/power_grid_model_ds/_core/model/grids/serialization/json.py

@@ -33,7 +33,22 @@ def serialize_to_json[G: Grid](grid: G, path: Path, strict: bool = True, **kwarg
         Path: The path where the file was saved
     """
     path.parent.mkdir(parents=True, exist_ok=True)
+    json_data = serialize_to_dict(grid=grid, strict=strict, **kwargs)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(json_data, f, **kwargs)
+    return path
+
 
+def serialize_to_dict[G: Grid](grid: G, strict: bool = True, **kwargs) -> dict:
+    """Serialize a Grid object to a Python dict.
+
+    Args:
+        grid: The Grid object to serialize
+        strict: Whether to raise an error if the grid object is not serializable.
+        **kwargs: Keyword arguments forwarded to json.dumps for serializability checks (e.g. cls).
+    Returns:
+        dict: A PGM-compatible dict representation of the grid.
+    """
     serialized_data = {}
 
     for field in dataclasses.fields(grid):
@@ -49,16 +64,10 @@ def serialize_to_json[G: Grid](grid: G, path: Path, strict: bool = True, **kwarg
         if _is_serializable(field_value, strict, **kwargs):
             serialized_data[field.name] = field_value
 
-    # Store in a wrapper for PGM compatibility
-    json_data = {"data": serialized_data}
-
-    with Path(path).open("w", encoding="utf-8") as f:
-        json.dump(json_data, f, **kwargs)
-
-    return path
+    return {"data": serialized_data}
 
 
-def deserialize_from_json[G: Grid](path: Path, target_grid_class: type[G], **kwargs) -> G:
+def deserialize_from_json[G: Grid](path: Path, target_grid_class: type[G]) -> G:
     """Load a Grid object from JSON format with cross-type loading support.
 
     Args:
@@ -68,19 +77,54 @@ def deserialize_from_json[G: Grid](path: Path, target_grid_class: type[G], **kwa
     Returns:
         Grid: The deserialized Grid object of the specified target class
     """
-    if "decoder_cls" in kwargs:
-        kwargs["cls"] = kwargs.pop("decoder_cls")
+    with path.open(encoding="utf-8") as f:
+        json_data = json.load(f)
+    return deserialize_from_dict(data=json_data, target_grid_class=target_grid_class)
 
-    with Path(path).open("r", encoding="utf-8") as f:
-        json_data = json.load(f, **kwargs)
 
+def deserialize_from_dict[G: Grid](data: dict, target_grid_class: type[G]) -> G:
+    """Load a Grid object from a Python dict.
+
+    Args:
+        data: A dict as produced by ``serialize_to_dict``.
+        target_grid_class: Grid class to load into.
+
+    Returns:
+        Grid: The deserialized Grid object of the specified target class
+    """
     grid = target_grid_class.empty()
-    _restore_grid_values(grid, json_data["data"])
+    _restore_grid_values(grid, data["data"])
     grid.rebuild_ids()
     grid.rebuild_graphs()
     return grid
 
 
+def serialize_to_json_string[G: Grid](grid: G, strict: bool = True, **kwargs) -> str:
+    """Serialize a Grid to a JSON string (in memory, no file I/O).
+
+    Args:
+        grid: The Grid object to serialize.
+        strict: Whether to raise an error if the grid is not serializable.
+        **kwargs: Forwarded to json.dumps (e.g. indent, sort_keys, cls).
+    Returns:
+        str: A JSON string representation of the grid.
+    """
+    data = serialize_to_dict(grid=grid, strict=strict, **kwargs)
+    return json.dumps(data, **kwargs)
+
+
+def deserialize_from_json_string[G: Grid](json_string: str, target_grid_class: type[G]) -> G:
+    """Load a Grid from a JSON string.
+
+    Args:
+        json_string: A JSON string as produced by ``serialize_to_json_string``.
+        target_grid_class: Grid class to load into.
+    Returns:
+        Grid: The deserialized Grid object.
+    """
+    return deserialize_from_dict(data=json.loads(json_string), target_grid_class=target_grid_class)
+
+
 def _restore_grid_values[G: Grid](grid: G, json_data: dict) -> None:
     """Restore arrays to the grid."""
     for attr_name, attr_values in json_data.items():

src/power_grid_model_ds/_core/utils/misc.py

@@ -23,22 +23,34 @@ def is_sequence(seq):
     return isinstance(seq, Sequence)
 
 
-def get_inherited_attrs(cls: type, *private_attributes):
-    """
-    Get the attribute from the object and all its parents
-    """
+def get_public_annotations(cls: type):
+    """Get the public annotations for a class"""
+    # Note: include_extras=True for annotated types like NDArray3
+    class_attributes = get_type_hints(cls, include_extras=True)
+    return {attr: type_ for attr, type_ in class_attributes.items() if not attr.startswith("_")}
 
-    # The extras are needed for annotated types like NDArray3
-    retrieved_attributes = get_type_hints(cls, include_extras=True)
-    retrieved_attributes = {attr: type for attr, type in retrieved_attributes.items() if not attr.startswith("_")}
 
-    for private_attr in private_attributes:
-        for parent in reversed(list(cls.__mro__)):
-            attr_dict = retrieved_attributes.get(private_attr, {})
-            attr_dict.update(getattr(parent, private_attr, {}))
-            retrieved_attributes[private_attr] = attr_dict
+def combine_attribute_from_parent_classes[T: (dict, set)](cls: type, attribute_name: str, attribute_type: type[T]) -> T:
+    """Combine all versions of an attribute in the Method Resolution Order (mro) of a class into a single attribute
 
-    return retrieved_attributes
+    For dicts this means the dict is updated so that child classes override parent classes.
+    For sets this means the sets are unioned together.
+
+    Types other than dict and set are not supported
+    """
+    combined_attr = attribute_type()
+    for parent in reversed(list(cls.__mro__)):
+        parent_attr = getattr(parent, attribute_name, attribute_type())
+        if attribute_type is dict:
+            combined_attr.update(parent_attr)
+        elif attribute_type is set:
+            combined_attr |= parent_attr
+        else:
+            raise NotImplementedError(
+                f"Type {attribute_type} cannot combine inherited for attribute {attribute_name}. "
+                f"Only dict and set are currently supported."
+            )
+    return combined_attr
 
 
 def array_equal_with_nan(array1: np.ndarray, array2: np.ndarray) -> bool: